Setting the file. One moment.
List Urls · Competitor Analysis · browserbase/skills · Skills Docs
Back to the top of the page (opens in a new tab)
scripts/ list_urls.mjs
JavaScript · 90 lines · 3 KB
12
13 if (args. includes ( '--help' ) || args. includes ( '-h' ) || args. length === 0 ) {
14 console. error ( `Usage: node list_urls.mjs <directory> [--prefix <prefix>]
15
16 Reads all <prefix>_discovery_batch_*.json files from <directory>,
17 deduplicates URLs by domain, and outputs one URL per line to stdout.
18
19 Options:
20 --prefix <prefix> Batch file prefix (default: "competitor")
21 --help, -h Show this help message
22
23 Examples:
24 node list_urls.mjs /tmp
25 node list_urls.mjs /tmp --prefix competitor` );
26 process. exit (args. includes ( '--help' ) || args. includes ( '-h' ) ? 0 : 1 );
27 }
28
29 const dir = args[ 0 ];
30 const prefixIdx = args. indexOf ( '--prefix' );
31 const prefix = prefixIdx !== - 1 && args[prefixIdx + 1 ] ? args[prefixIdx + 1 ] : 'competitor' ;
32
33 // Escape regex metacharacters in the user-supplied prefix so a value like
34 // "comp.+" matches the literal filename, not as a regex pattern.
35 const escapedPrefix = prefix. replace ( / [.*+?^${}()|[ \]\\ ] / g , ' \\ $&' );
36 const pattern = new RegExp ( `^${ escapedPrefix }_discovery_batch_.* \\ .json$` );
37
38 let files;
39 try {
40 files = readdirSync (dir)
41 . filter ( f => pattern. test (f))
42 . sort ();
43 } catch (err) {
44 console. error ( `Error reading directory ${ dir }: ${ err . message }` );
45 process. exit ( 1 );
46 }
47
48 if (files. length === 0 ) {
49 console. error ( `No ${ prefix }_discovery_batch_*.json files found in ${ dir }` );
50 process. exit ( 1 );
51 }
52
53 // Dedup by hostname, but prefer the site root over a deep link. The first search hit for a
54 // domain is often a blog/doc/comparison path; gating + enrichment want the homepage, so when
55 // multiple URLs share a host we keep the shallowest path (fewest segments). First-seen host
56 // order is preserved (Map.set on an existing key keeps its position).
57 const byDomain = new Map (); // hostname -> { url, depth }
58 let totalResults = 0 ;
59
60 for ( const file of files) {
61 try {
62 const data = JSON . parse ( readFileSync ( join (dir, file), 'utf-8' ));
63 const results = Array. isArray (data) ? data : (data.results || []);
64 totalResults += results. length ;
65
66 for ( const result of results) {
67 const url = result.url;
68 if ( ! url) continue ;
69
70 try {
71 const u = new URL (url);
72 const hostname = u.hostname. replace ( / ^ www \. / , '' );
73 const depth = u.pathname. replace ( / \/ +$ / , '' ). split ( '/' ). filter (Boolean). length ;
74 const existing = byDomain. get (hostname);
75 if ( ! existing || depth < existing.depth) byDomain. set (hostname, { url, depth });
76 } catch {
77 // Skip invalid URLs
78 }
79 }
80 } catch (err) {
81 console. error ( `Warning: Failed to parse ${ file }: ${ err . message }` );
82 }
83 }
84
85 const urls = [ ... byDomain. values ()]. map ( v => v.url);
86 for ( const url of urls) {
87 console. log (url);
88 }
89
90 console. error ( ` \n ${ files . length } files, ${ totalResults } total results, ${ urls . length } unique domains` );