Setting the file. One moment. 28.10
Workflow
This file
- Number
- 28.38
- Position
- 38 of 89
- Type
- JavaScript
- Size
- 9 KB
- Lines
- 329
scripts/lib/common.mjs
JavaScript·329 lines·9 KB
, height:
844
},
8];
9
10export const DISCOVERY_LIMITS = {
11 home: { maxUrls: 1, maxDepth: 0 },
12 blog: { maxUrls: 50, maxDepth: 3, representativePosts: 3 },
13 ecommerce: { maxUrls: 75, maxDepth: 3, representativeProducts: 3, representativeCategories: 3 },
14 full: { maxUrls: 150, maxDepth: 3 },
15 specific: { maxUrls: 150, maxDepth: 1 },
16};
17
18export function parseArgs(argv = process.argv.slice(2)) {
19 const args = { _: [] };
20 for (let i = 0; i < argv.length; i += 1) {
21 const arg = argv[i];
22 if (!arg.startsWith("--")) {
23 args._.push(arg);
24 continue;
25 }
26 const eq = arg.indexOf("=");
27 if (eq !== -1) {
28 args[arg.slice(2, eq)] = arg.slice(eq + 1);
29 continue;
30 }
31 const key = arg.slice(2);
32 const next = argv[i + 1];
33 if (!next || next.startsWith("--")) {
34 args[key] = true;
35 continue;
36 }
37 args[key] = next;
38 i += 1;
39 }
40 return args;
41}
42
43export function normalizeUrl(value) {
44 if (!value) throw new Error("Missing URL");
45 const withProtocol = /^https?:\/\//i.test(value) ? value : `https://${value}`;
46 const url = new URL(withProtocol);
47 url.hash = "";
48 return url;
49}
50
51export function sameOrigin(sourceUrl, candidate) {
52 try {
53 return new URL(candidate, sourceUrl).origin === new URL(sourceUrl).origin;
54 } catch {
55 return false;
56 }
57}
58
59export function normalizeDiscoveredUrl(sourceUrl, href) {
60 try {
61 const url = new URL(href, sourceUrl);
62 if (!/^https?:$/.test(url.protocol)) return null;
63 url.hash = "";
64 if (url.pathname !== "/" && url.pathname.endsWith("/")) {
65 url.pathname = url.pathname.slice(0, -1);
66 }
67 return url.toString();
68 } catch {
69 return null;
70 }
71}
72
73const NON_PAGE_EXTENSIONS = new Set([
74 ".jpg",
75 ".jpeg",
76 ".png",
77 ".gif",
78 ".webp",
79 ".svg",
80 ".ico",
81 ".bmp",
82 ".avif",
83 ".mp4",
84 ".webm",
85 ".mov",
86 ".m4v",
87 ".mp3",
88 ".wav",
89 ".ogg",
90 ".pdf",
91 ".zip",
92 ".rar",
93 ".7z",
94 ".tar",
95 ".gz",
96 ".xml",
97 ".json",
98 ".txt",
99 ".css",
100 ".js",
101 ".mjs",
102 ".map",
103 ".woff",
104 ".woff2",
105 ".ttf",
106 ".otf",
107 ".eot",
108]);
109
110export function isAssetLikeUrl(urlValue) {
111 try {
112 const url = new URL(urlValue);
113 const pathname = url.pathname.toLowerCase();
114 const extension = path.extname(pathname);
115 if (NON_PAGE_EXTENSIONS.has(extension)) return true;
116 return pathname.includes("/wp-content/uploads/");
117 } catch {
118 return false;
119 }
120}
121
122export function projectNameFromUrl(urlValue) {
123 const url = normalizeUrl(urlValue);
124 let host = url.hostname.toLowerCase().replace(/^www\./, "");
125 const parts = host.split(".");
126 if (parts.length === 2) host = parts[0];
127 return kebab(host);
128}
129
130export function kebab(value) {
131 return String(value || "site")
132 .toLowerCase()
133 .replace(/[^a-z0-9]+/g, "-")
134 .replace(/^-+|-+$/g, "")
135 .replace(/-{2,}/g, "-") || "site";
136}
137
138export function resolveOutputDir(urlValue, out) {
139 if (out) return path.normalize(out);
140 return path.join("projects", projectNameFromUrl(urlValue));
141}
142
143export function docsDir(outputDir) {
144 return path.join(outputDir, "docs", "site-clone");
145}
146
147export async function ensureDir(dir) {
148 await mkdir(dir, { recursive: true });
149}
150
151export async function writeJson(filePath, data) {
152 await ensureDir(path.dirname(filePath));
153 await writeFile(filePath, `${JSON.stringify(data, null, 2)}\n`, "utf8");
154}
155
156export async function readJson(filePath) {
157 return JSON.parse(await readFile(filePath, "utf8"));
158}
159
160export async function writeText(filePath, data) {
161 await ensureDir(path.dirname(filePath));
162 await writeFile(filePath, data, "utf8");
163}
164
165export async function fetchText(url, options = {}) {
166 const timeoutMs = Number(options.timeoutMs || 5000);
167 const response = await fetch(url, {
168 redirect: "follow",
169 signal: AbortSignal.timeout(timeoutMs),
170 headers: {
171 "user-agent": "site-clone-skill/1.0 (+https://wix-headless.dev)",
172 accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
173 ...options.headers,
174 },
175 });
176 if (!response.ok) throw new Error(`Fetch failed ${response.status} for ${url}`);
177 return response.text();
178}
179
180export function extractLinks(html, baseUrl) {
181 const links = [];
182 const pattern = /<a\b[^>]*href\s*=\s*["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
183 let match;
184 while ((match = pattern.exec(html))) {
185 const url = normalizeDiscoveredUrl(baseUrl, match[1]);
186 if (!url) continue;
187 links.push({
188 url,
189 text: stripTags(match[2]).replace(/\s+/g, " ").trim(),
190 });
191 }
192 return dedupeBy(links, (link) => link.url);
193}
194
195export function stripTags(value) {
196 return String(value || "")
197 .replace(/<script[\s\S]*?<\/script>/gi, "")
198 .replace(/<style[\s\S]*?<\/style>/gi, "")
199 .replace(/<[^>]+>/g, " ")
200 .replace(/ /g, " ")
201 .replace(/&/g, "&")
202 .replace(/</g, "<")
203 .replace(/>/g, ">")
204 .trim();
205}
206
207export function classifyUrl(urlValue) {
208 const url = new URL(urlValue);
209 const pathValue = url.pathname.toLowerCase();
210 const joined = `${pathValue} ${url.search.toLowerCase()}`;
211 if (isAssetLikeUrl(urlValue)) return "asset";
212 if (pathValue === "/" || pathValue === "") return "home";
213 if (/(\/products?\/|\/product\/|\/shop\/[^/]+|[?&]product=)/.test(joined)) return "product";
214 if (/(\/collections?\/|\/categories?\/|\/category\/|\/shop\/?$|\/store\/?$)/.test(joined)) return "product-category";
215 if (/(\/blog\/[^/]+|\/posts?\/[^/]+|\/article\/)/.test(joined)) return "blog-post";
216 if (/(\/blog\/?$|\/news\/?$|\/articles\/?$)/.test(joined)) return "blog-index";
217 if (/(\/booking|\/bookings|\/appointments?|\/schedule)/.test(joined)) return "bookings";
218 if (/(\/events?\/|\/event\/)/.test(joined)) return "events";
219 if (/(\/pricing|\/plans|\/membership)/.test(joined)) return "pricing";
220 if (/(\/docs|\/documentation|\/help|\/support|\/kb)/.test(joined)) return "docs";
221 if (/(\/about|\/contact|\/team|\/company)/.test(joined)) return "about-contact";
222 if (/(\/privacy|\/terms|\/legal|\/cookies?)/.test(joined)) return "legal";
223 if (/(\/cms|\/resources?|\/guides?|\/case-stud)/.test(joined)) return "cms-content";
224 return "other";
225}
226
227export function classifyTemplate(area) {
228 const dynamic = {
229 product: "product-detail",
230 "product-category": "product-category",
231 "blog-post": "blog-post",
232 "blog-index": "blog-index",
233 "cms-content": "cms-item",
234 bookings: "booking",
235 events: "event",
236 pricing: "pricing-plan",
237 };
238 return dynamic[area] || area;
239}
240
241export function isDynamicArea(area) {
242 return [
243 "product",
244 "product-category",
245 "blog-post",
246 "blog-index",
247 "cms-content",
248 "bookings",
249 "events",
250 "pricing",
251 ].includes(area);
252}
253
254export function countBy(items, keyFn) {
255 const counts = {};
256 for (const item of items) {
257 const key = keyFn(item);
258 counts[key] = (counts[key] || 0) + 1;
259 }
260 return counts;
261}
262
263export function dedupeBy(items, keyFn) {
264 const seen = new Set();
265 const out = [];
266 for (const item of items) {
267 const key = keyFn(item);
268 if (seen.has(key)) continue;
269 seen.add(key);
270 out.push(item);
271 }
272 return out;
273}
274
275export function slugForUrl(urlValue) {
276 const url = new URL(urlValue);
277 return kebab(`${url.hostname}${url.pathname === "/" ? "-home" : url.pathname}`);
278}
279
280export function routePathFromUrl(sourceUrl, targetUrl) {
281 const url = new URL(targetUrl, sourceUrl);
282 return `${url.pathname}${url.search}` || "/";
283}
284
285export function selectWixTemplate(scope, counts = {}) {
286 if (scope === "ecommerce" || counts.product || counts["product-category"]) return "commerce";
287 if (counts.bookings) return "scheduler";
288 if (counts.events || counts.pricing) return "registration";
289 return "blank";
290}
291
292export function businessNameFromProject(projectName) {
293 return projectName
294 .split(/[-_\s]+/)
295 .filter(Boolean)
296 .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
297 .join(" ");
298}
299
300export function wixFolderNameFromProject(projectName) {
301 return kebab(projectName);
302}
303
304export function wixBusinessNameFromProject(projectName) {
305 const value = businessNameFromProject(wixFolderNameFromProject(projectName));
306 return value || "Site";
307}
308
309export function limitRepresentativePages(pages, scope) {
310 const limits = DISCOVERY_LIMITS[scope] || DISCOVERY_LIMITS.full;
311 const counts = {};
312 return pages.filter((page) => {
313 if (page.area === "asset") return false;
314 const area = page.area;
315 if (area === "product") {
316 counts.product = (counts.product || 0) + 1;
317 return counts.product <= (limits.representativeProducts || 3);
318 }
319 if (area === "product-category") {
320 counts["product-category"] = (counts["product-category"] || 0) + 1;
321 return counts["product-category"] <= (limits.representativeCategories || 3);
322 }
323 if (area === "blog-post") {
324 counts["blog-post"] = (counts["blog-post"] || 0) + 1;
325 return counts["blog-post"] <= (limits.representativePosts || 3);
326 }
327 return true;
328 });
329}