Setting the file. One moment. 28.10
Workflow
scripts/discover-urls.mjs
scripts/discover-urls.mjs
JavaScript·142 lines·4 KB
parseArgs,
16 resolveOutputDir,
17 sameOrigin,
18 writeJson,
19} from "./lib/common.mjs";
20
21async function main() {
22 const args = parseArgs();
23 const sourceUrl = normalizeUrl(args._[0] || args.url).toString();
24 const scope = String(args.scope || "home");
25 const outputDir = resolveOutputDir(sourceUrl, args.out);
26 const outDir = docsDir(outputDir);
27 await ensureDir(outDir);
28
29 const explicitUrls = parseUrlList(args.urls, sourceUrl);
30 const discovery = await discover({ sourceUrl, scope, explicitUrls });
31 await writeJson(path.join(outDir, "discovery.json"), discovery);
32
33 if (args.json) {
34 process.stdout.write(`${JSON.stringify(discovery, null, 2)}\n`);
35 return;
36 }
37 printSummary(discovery);
38}
39
40export async function discover({ sourceUrl, scope = "home", explicitUrls = [] }) {
41 const normalizedScope = explicitUrls.length ? "specific" : scope;
42 const limits = DISCOVERY_LIMITS[normalizedScope] || DISCOVERY_LIMITS.full;
43 const source = normalizeUrl(sourceUrl).toString();
44 const seedUrls = new Set([source, ...explicitUrls]);
45 const queue = [{ url: source, depth: 0, inScope: true }];
46 for (const url of explicitUrls) queue.push({ url, depth: 0 });
47
48 const seen = new Set();
49 const pages = [];
50 const excluded = [];
51
52 while (queue.length && pages.length < limits.maxUrls) {
53 const current = queue.shift();
54 if (!current || seen.has(current.url)) continue;
55 seen.add(current.url);
56 if (!sameOrigin(source, current.url)) {
57 excluded.push({ url: current.url, reason: "external" });
58 continue;
59 }
60 if (isAssetLikeUrl(current.url)) {
61 excluded.push({ url: current.url, reason: "non-page-asset" });
62 continue;
63 }
64
65 const area = classifyUrl(current.url);
66 pages.push({
67 url: current.url,
68 path: new URL(current.url).pathname || "/",
69 area,
70 depth: current.depth,
71 inScope: current.inScope ?? (seedUrls.has(current.url) || normalizedScope !== "specific"),
72 });
73
74 if (normalizedScope === "home" || current.depth >= limits.maxDepth) continue;
75
76 try {
77 const html = await fetchText(current.url);
78 const links = extractLinks(html, current.url);
79 for (const link of links) {
80 if (pages.length + queue.length >= limits.maxUrls * 2) break;
81 if (!sameOrigin(source, link.url)) {
82 excluded.push({ url: link.url, reason: "external" });
83 continue;
84 }
85 if (isAssetLikeUrl(link.url)) {
86 excluded.push({ url: link.url, reason: "non-page-asset" });
87 continue;
88 }
89 if (!seen.has(link.url)) queue.push({ url: link.url, depth: current.depth + 1, inScope: normalizedScope !== "specific" });
90 }
91 } catch (error) {
92 excluded.push({ url: current.url, reason: `fetch-failed: ${error.message}` });
93 }
94 }
95
96 const inScopePages = pages.filter((page) => page.inScope);
97 const preservedPages = pages.filter((page) => !page.inScope);
98 const representativePages = normalizedScope === "specific"
99 ? inScopePages
100 : limitRepresentativePages(pages, normalizedScope);
101 return {
102 sourceUrl: source,
103 scope: normalizedScope,
104 limits,
105 totalDiscovered: pages.length,
106 countsByArea: countBy(pages, (page) => page.area),
107 pages,
108 inScopePages,
109 preservedPages,
110 representativePages,
111 excluded,
112 requiresConfirmation: normalizedScope !== "home" && normalizedScope !== "specific",
113 generatedAt: new Date().toISOString(),
114 };
115}
116
117function parseUrlList(value, sourceUrl) {
118 if (!value) return [];
119 return String(value)
120 .split(",")
121 .map((item) => normalizeDiscoveredUrl(sourceUrl, item.trim()))
122 .filter((item) => item && !isAssetLikeUrl(item))
123 .filter(Boolean);
124}
125
126function printSummary(discovery) {
127 console.log(`Discovered ${discovery.totalDiscovered} same-origin URL(s).`);
128 for (const [area, count] of Object.entries(discovery.countsByArea)) {
129 console.log(`- ${area}: ${count}`);
130 }
131 if (discovery.requiresConfirmation) {
132 console.log("");
133 console.log("Confirmation required before extraction/building.");
134 }
135}
136
137if (import.meta.url === `file://${process.argv[1]}`) {
138 main().catch((error) => {
139 console.error(error.stack || error.message);
140 process.exit(1);
141 });
142}