Setting the file. One moment. 28.10
Workflow
scripts/extract-assets.mjs
JavaScript·140 lines·5 KB
14} from "./lib/common.mjs";
15import { extractAssetsFromHtml, extractStylesheetUrls } from "./lib/html-extract.mjs";
16import { buildFontManifest } from "./lib/font-contract.mjs";
17
18async function main() {
19 const args = parseArgs();
20 const url = normalizeUrl(args._[0] || args.url).toString();
21 const outputDir = resolveOutputDir(url, args.out);
22 const manifest = await extractAssets(url, { outputDir, download: args.download !== "false" });
23 await writeJson(path.join(docsDir(outputDir), "assets.json"), manifest);
24 await writeJson(path.join(docsDir(outputDir), "fonts.json"), manifest.fonts);
25 if (args.json) process.stdout.write(`${JSON.stringify(manifest, null, 2)}\n`);
26}
27
28export async function extractAssets(url, { outputDir, download = true } = {}) {
29 const html = await fetchText(url);
30 const stylesheets = await loadStylesheets(html, url);
31 const fonts = buildFontManifest({ html, baseUrl: url, stylesheets });
32 const discovered = dedupeBy(
33 [
34 ...extractAssetsFromHtml(html, url),
35 ...fonts.faces.flatMap((face) =>
36 (face.sources || [])
37 .filter((source) => source.kind === "url" && source.url)
38 .map((source) => ({ sourceUrl: source.url, type: "font-face" })),
39 ),
40 ].filter((asset) => usefulAsset(asset.sourceUrl)),
41 (asset) => asset.sourceUrl,
42 );
43 const assets = [];
44 for (const asset of discovered.slice(0, 120)) {
45 const localPath = path.join("public", "site-clone", "assets", localAssetName(asset.sourceUrl));
46 const record = { ...asset, localPath };
47 if (download && outputDir) {
48 try {
49 await downloadAsset(asset.sourceUrl, path.join(outputDir, localPath));
50 record.downloaded = true;
51 } catch (error) {
52 record.downloaded = false;
53 record.error = error.message;
54 }
55 }
56 assets.push(record);
57 }
58 const assetByUrl = new Map(assets.map((asset) => [asset.sourceUrl, asset]));
59 const hydratedFonts = {
60 ...fonts,
61 faces: fonts.faces.map((face) => ({
62 ...face,
63 sources: (face.sources || []).map((source) => {
64 const downloaded = assetByUrl.get(source.url);
65 return downloaded
66 ? {
67 ...source,
68 localPath: downloaded.localPath,
69 downloaded: downloaded.downloaded === true,
70 }
71 : source;
72 }),
73 })),
74 };
75 return { url, assets, fonts: hydratedFonts, generatedAt: new Date().toISOString() };
76}
77
78async function loadStylesheets(html, url) {
79 const urls = extractStylesheetUrls(html, url);
80 const stylesheets = [];
81 for (const stylesheetUrl of urls.slice(0, 24)) {
82 try {
83 const cssText = await fetchText(stylesheetUrl, {
84 headers: {
85 accept: "text/css,*/*;q=0.1",
86 },
87 });
88 stylesheets.push({ url: stylesheetUrl, cssText, sourceType: "linked-stylesheet" });
89 } catch {
90 // Preserve recoverability; unresolved stylesheets stay out of the deterministic manifest.
91 }
92 }
93 return stylesheets;
94}
95
96async function downloadAsset(url, filePath) {
97 const response = await fetch(url, { redirect: "follow" });
98 if (!response.ok) throw new Error(`Fetch failed ${response.status}`);
99 const buffer = Buffer.from(await response.arrayBuffer());
100 await ensureDir(path.dirname(filePath));
101 await import("node:fs/promises").then(({ writeFile }) => writeFile(filePath, buffer));
102}
103
104function localAssetName(urlValue) {
105 const pathname = assetPathname(urlValue);
106 const ext = path.extname(pathname).split("?")[0] || ".bin";
107 return `${assetSlug(urlValue).slice(0, 70)}-${createHash("sha1").update(urlValue).digest("hex").slice(0, 8)}${ext}`;
108}
109
110function usefulAsset(urlValue) {
111 return /\.(png|jpe?g|webp|gif|svg|ico|avif|mp4|webm|woff2?|ttf|otf)(\?|$)/i.test(urlValue);
112}
113
114function assetPathname(urlValue) {
115 try {
116 return new URL(urlValue).pathname || "";
117 } catch {
118 try {
119 return new URL(urlValue, "https://local.invalid").pathname || "";
120 } catch {
121 const sanitized = String(urlValue || "").split("#")[0].split("?")[0];
122 return sanitized.startsWith("/") ? sanitized : `/${sanitized}`;
123 }
124 }
125}
126
127function assetSlug(urlValue) {
128 try {
129 return slugForUrl(urlValue);
130 } catch {
131 return slugForUrl(`https://local.invalid${assetPathname(urlValue) || "/asset"}`);
132 }
133}
134
135if (import.meta.url === `file://${process.argv[1]}`) {
136 main().catch((error) => {
137 console.error(error.stack || error.message);
138 process.exit(1);
139 });
140}