List Urls

1#!/usr/bin/env node
2
3// Deduplicates discovery URLs from browse cloud search JSON output files.
4// Usage: node list_urls.mjs /tmp [--prefix company]
5// Reads all {prefix}_discovery_batch_*.json files, deduplicates by domain,
6// outputs one URL per line to stdout, stats to stderr.
7
8import { readdirSync, readFileSync } from 'fs';
9import { join } from 'path';
10
11const args = process.argv.slice(2);
12
13if (args.includes('--help') || args.includes('-h') || args.length === 0) {
14  console.error(`Usage: node list_urls.mjs <directory> [--prefix <prefix>]
15
16Reads all <prefix>_discovery_batch_*.json files from <directory>,
17deduplicates URLs by domain, and outputs one URL per line to stdout.
18
19Options:
20  --prefix <prefix>  Batch file prefix (default: "company")
21  --help, -h         Show this help message
22
23Examples:
24  node list_urls.mjs /tmp
25  node list_urls.mjs /tmp --prefix company`);
26  process.exit(args.includes('--help') || args.includes('-h') ? 0 : 1);
27}
28
29const dir = args[0];
30const prefixIdx = args.indexOf('--prefix');
31const prefix = prefixIdx !== -1 && args[prefixIdx + 1] ? args[prefixIdx + 1] : 'company';
32
33const pattern = new RegExp(`^${prefix}_discovery_batch_.*\\.json$`);
34
35let files;
36try {
37  files = readdirSync(dir)
38    .filter(f => pattern.test(f))
39    .sort();
40} catch (err) {
41  console.error(`Error reading directory ${dir}: ${err.message}`);
42  process.exit(1);
43}
44
45if (files.length === 0) {
46  console.error(`No ${prefix}_discovery_batch_*.json files found in ${dir}`);
47  process.exit(1);
48}
49
50const seenDomains = new Set();
51const urls = [];
52let totalResults = 0;
53
54for (const file of files) {
55  try {
56    const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
57    const results = Array.isArray(data) ? data : (data.results || []);
58    totalResults += results.length;
59
60    for (const result of results) {
61      const url = result.url;
62      if (!url) continue;
63
64      try {
65        const hostname = new URL(url).hostname.replace(/^www\./, '');
66        if (!seenDomains.has(hostname)) {
67          seenDomains.add(hostname);
68          urls.push(url);
69        }
70      } catch {
71        // Skip invalid URLs
72      }
73    }
74  } catch (err) {
75    console.error(`Warning: Failed to parse ${file}: ${err.message}`);
76  }
77}
78
79// Output deduplicated URLs to stdout
80for (const url of urls) {
81  console.log(url);
82}
83
84// Stats to stderr
85console.error(`\n${files.length} files, ${totalResults} total results, ${urls.length} unique domains`);