Setting the file. One moment.
Mirror Site · Wix Harmony Editor · wix/skills · Skills Docs
Back to the top of the page scripts/copy/mirror-site.mjs
scripts/copy/ mirror-site.mjs
JavaScript · 92 lines · 5 KB
{ chromium }
from
'playwright'
;
14 import fs from 'node:fs' ;
15 import crypto from 'node:crypto' ;
16
17 const args = process.argv. slice ( 2 );
18 const opt = ( n , d ) => { const i = args. indexOf ( `--${ n }` ); return i >= 0 ? args[i + 1 ] : d; };
19 const pos = args. filter (( a , i ) => ! a. startsWith ( '--' ) && ! (i > 0 && args[i - 1 ]. startsWith ( '--' )));
20 const [ outDir , ... urls ] = pos;
21 if ( ! outDir || ! urls. length ) { console. error ( 'usage: mirror-site.mjs <outDir> <url...> [--widths 390,1024,1280,1920] [--resume]' ); process. exit ( 2 ); }
22 const widths = opt ( 'widths' , '390,1024,1280,1920' ). split ( ',' ). map (Number);
23 fs. mkdirSync ( `${ outDir }/bodies` , { recursive: true });
24 const idxFile = `${ outDir }/index.json` ;
25 const index = fs. existsSync (idxFile) ? JSON . parse (fs. readFileSync (idxFile, 'utf8' )) : { site: new URL (urls[ 0 ]).host, pages: [], resources: {} };
26 index.embeds ??= {};
27 const key = ( u ) => u. split ( '#' )[ 0 ];
28 const failures = [];
29
30 const browser = await chromium. launch ();
31 async function capture ( url , width ) {
32 const ctx = await browser. newContext ({ viewport: { width, height: 900 } });
33 const page = await ctx. newPage ();
34 const pending = [];
35 let ok = true ;
36 page. on ( 'response' , ( res ) => pending. push (( async () => {
37 const u = key (res. url ());
38 if ( ! / ^ https ? :/ . test (u)) return ;
39 const status = res. status ();
40 const headers = res. headers ();
41 if (status >= 300 && status < 400 ) { index.resources[u] ??= { status, location: headers.location || '' }; return ; }
42 let body; try { body = await res. body (); } catch { return ; } // e.g. aborted beacons
43 const sha = crypto. createHash ( 'sha1' ). update (body). digest ( 'hex' );
44 if ( ! fs. existsSync ( `${ outDir }/bodies/${ sha }` )) fs. writeFileSync ( `${ outDir }/bodies/${ sha }` , body);
45 const prev = index.resources[u];
46 if ( ! prev || prev.status !== 200 || status === 200 ) index.resources[u] = { status, type: headers[ 'content-type' ] || '' , body: sha, bytes: body. length };
47 })()));
48 try {
49 await page. goto (url, { waitUntil: 'networkidle' , timeout: 90000 });
50 // scroll through the page so lazy-loaded images and scripts load, then back to the top
51 const h = await page. evaluate (() => document.documentElement.scrollHeight);
52 for ( let y = 0 ; y <= h; y += 600 ) { await page. evaluate (( y ) => scrollTo ( 0 , y), y); await page. waitForTimeout ( 120 ); }
53 await page. evaluate (() => scrollTo ( 0 , 0 ));
54 await page. waitForLoadState ( 'networkidle' ). catch (() => {});
55 await page. waitForTimeout ( 800 );
56 if (width === 1280 ) {
57 for ( const el of await page. $$ ( 'iframe[src]' )) {
58 const src = await el. getAttribute ( 'src' );
59 if ( ! / ^ https ? :/ . test (src) || new URL (src).host === index.site) continue ;
60 const box = await el. evaluate (( e ) => { e. scrollIntoView ({ block: 'center' }); const r = e. getBoundingClientRect (); return { x: r.x + scrollX + e.clientLeft, y: r.y + scrollY + e.clientTop, width: e.clientWidth, height: e.clientHeight }; }); // content box: the border stays CSS
61 if (box.width < 2 || box.height < 2 ) continue ;
62 await page. waitForTimeout ( 3000 ); // map tiles
63 const png = await page. screenshot ({ fullPage: true , clip: box });
64 const sha = crypto. createHash ( 'sha1' ). update (png). digest ( 'hex' );
65 fs. writeFileSync ( `${ outDir }/bodies/${ sha }` , png);
66 index.embeds[src] = { page: key (url), body: sha };
67 }
68 await page. evaluate (() => scrollTo ( 0 , 0 ));
69 }
70 } catch (e) { ok = false ; failures. push ({ url, width, error: String (e). slice ( 0 , 200 ) }); }
71 await Promise . all (pending);
72 await ctx. close ();
73 return ok;
74 }
75 // a long list is saved as it goes (every 10 pages) and --resume skips pages already in the index, so a crash or a
76 // stopped run costs minutes, not the whole capture
77 const resume = args. includes ( '--resume' );
78 let done = 0 ;
79 for ( const url of urls) {
80 if (resume && index.captured?. includes ( key (url))) continue ;
81 if ( ! index.pages. includes ( key (url))) index.pages. push ( key (url));
82 const results = await Promise . all (widths. map (( w ) => capture (url, w)));
83 // only a page captured at every width counts as captured, so --resume retries a failed one
84 if ( ! results. every (Boolean)) continue ;
85 (index.captured ??= []). push ( key (url));
86 if ( ++ done % 10 === 0 ) fs. writeFileSync (idxFile, JSON . stringify (index));
87 }
88 await browser. close ();
89 fs. writeFileSync (idxFile, JSON . stringify (index, null , 1 ));
90 const res = Object. values (index.resources);
91 const hosts = [ ...new Set (Object. keys (index.resources). map (( u ) => new URL (u).host))];
92 console. log ( JSON . stringify ({ pages: index.pages. length , embeds: Object. keys (index.embeds). length , resources: res. length , bytes: res. reduce (( s , r ) => s + (r.bytes || 0 ), 0 ), hosts, failures }));