Merge Mirrors

1#!/usr/bin/env node
2// Merge mirrors captured in parallel (mirror-site.mjs into separate directories) into one.
3//
4//   node scripts/copy/merge-mirrors.mjs <intoDir> <fromDir> [<fromDir> ...]
5//
6// A capture takes ~25 s per page: split a long URL list over several directories, run them at once, merge.
7// resources / embeds: the first entry for a URL wins; pages and captured are unioned; bodies (named by hash)
8// are copied when missing. <intoDir> may be one of the sources' ancestors' mirror: it is created if absent.
9import fs from 'node:fs';
10
11const [into, ...from] = process.argv.slice(2);
12if (!into || !from.length) { console.error('usage: merge-mirrors.mjs <intoDir> <fromDir...>'); process.exit(2); }
13fs.mkdirSync(`${into}/bodies`, { recursive: true });
14const file = `${into}/index.json`;
15const out = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : { site: null, pages: [], resources: {}, embeds: {} };
16out.embeds ??= {}; out.captured ??= [];
17let copied = 0, added = 0;
18for (const dir of from) {
19  const idx = JSON.parse(fs.readFileSync(`${dir}/index.json`, 'utf8'));
20  out.site ??= idx.site;
21  if (idx.site !== out.site) { console.error(`${dir} mirrors ${idx.site}, not ${out.site}`); process.exit(1); }
22  for (const [u, r] of Object.entries(idx.resources)) if (!out.resources[u]) { out.resources[u] = r; added++; }
23  for (const [u, e] of Object.entries(idx.embeds || {})) out.embeds[u] ??= e;
24  out.pages = [...new Set([...out.pages, ...idx.pages])];
25  out.captured = [...new Set([...out.captured, ...(idx.captured || [])])];
26  if (fs.existsSync(`${dir}/bodies`)) for (const b of fs.readdirSync(`${dir}/bodies`)) if (!fs.existsSync(`${into}/bodies/${b}`)) { fs.copyFileSync(`${dir}/bodies/${b}`, `${into}/bodies/${
27}
28fs.writeFileSync(file, JSON.stringify(out));
29console.log(JSON.stringify({ into, pages: out.pages.length, captured: out.captured.length, resources: Object.keys(out.resources).length, newResources: added, bodiesCopied: copied }));