Setting the file. One moment.
Filter · Browser To API · browserbase/skills · Skills Docs
Back to the top of the page (opens in a new tab)
scripts/ filter.mjs
JavaScript · 76 lines · 3 KB
com)/
i
,
13 /mixpanel \. com/ i ,
14 /google-analytics \. com/ i ,
15 /googletagmanager \. com/ i ,
16 /datadog(hq) ? \. com/ i ,
17 /sentry \. io/ i ,
18 /amplitude \. com/ i ,
19 /fullstory \. com/ i ,
20 /hotjar \. com/ i ,
21 /intercom \. io/ i ,
22 /clarity \. ms/ i ,
23 /cloudflareinsights \. com/ i ,
24 /doubleclick \. net/ i ,
25 /facebook \. com \/ tr/ i ,
26 // Static assets
27 / \. (png | jpe ? g | gif | svg | webp | ico | woff2 ?| ttf | eot | otf | css | map | mp4 | webm | mp3 | m4a)( \? |$ )/ i ,
28 // SW / metadata
29 / \/ sw \. js( \? |$ )/ i ,
30 / \/ service-worker \. js( \? |$ )/ i ,
31 / \/ manifest \. json( \? |$ )/ i ,
32 / \/ robots \. txt( \? |$ )/ i ,
33 / \/ favicon \. ico( \? |$ )/ i ,
34 ];
35
36 export function filter ( outDir , opts = {}) {
37 const { include = [], exclude = [], origins = [] } = opts;
38 // Precedence:
39 // 1. --origins gates everything; non-matching is dropped.
40 // 2. User --exclude always wins (explicit user intent).
41 // 3. Default excludes can be rescued by --include (REFERENCE.md contract).
42 // 4. When --include is set, anything that doesn't match it is dropped.
43 const userExcludeRes = exclude. map ( s => new RegExp (s));
44 const includeRes = include. map ( s => new RegExp (s));
45 const originSet = new Set (origins);
46
47 const paired = readJsonl ( intermediatePath (outDir, 'paired.jsonl' ));
48 const out = [];
49 let droppedOrigin = 0 , droppedExclude = 0 , droppedInclude = 0 ;
50
51 for ( const row of paired) {
52 if (originSet.size) {
53 const host = row.origin ? new URL (row.origin).host : '' ;
54 const matched = [ ... originSet]. some ( o => host === o || host. endsWith ( '.' + o));
55 if ( ! matched) { droppedOrigin ++ ; continue ; }
56 }
57 if (userExcludeRes. some ( re => re. test (row.url))) { droppedExclude ++ ; continue ; }
58
59 const matchesInclude = includeRes. length > 0 && includeRes. some ( re => re. test (row.url));
60 const matchesDefaultExclude = DEFAULT_EXCLUDES . some ( re => re. test (row.url));
61 if (matchesDefaultExclude && ! matchesInclude) { droppedExclude ++ ; continue ; }
62 if (includeRes. length && ! matchesInclude) { droppedInclude ++ ; continue ; }
63
64 out. push (row);
65 }
66
67 writeJsonl ( intermediatePath (outDir, 'filtered.jsonl' ), out);
68 return { kept: out. length , droppedOrigin, droppedExclude, droppedInclude };
69 }
70
71 if ( import . meta .url === `file://${ process . argv [ 1 ] }` ) {
72 const out = process.argv[ 2 ];
73 if ( ! out) { console. error ( 'usage: filter.mjs <out-dir>' ); process. exit ( 2 ); }
74 const stats = filter (out);
75 console. log ( `filter: kept ${ stats . kept }, dropped ${ stats . droppedExclude } (exclude) ${ stats . droppedOrigin } (origin) ${ stats . droppedInclude } (include)` );
76 }