Setting the file. One moment.
Gate Candidates · Competitor Analysis · browserbase/skills · Skills Docs
Back to the top of the page scripts/ gate_candidates.mjs
JavaScript · 194 lines · 9 KB
15
// "matched_includes": [...], "matched_excludes": [...], "title": "...", "hero": "..." }
16
17 import { execFile } from 'child_process' ;
18 import { promisify } from 'util' ;
19 import { readFileSync } from 'fs' ;
20
21 // Async execFile so the worker pool actually parallelizes. spawnSync blocks the entire
22 // event loop, which silently turns --concurrency N into N=1 — every URL fetched serially
23 // regardless of the flag. With promisified execFile, N workers can wait on N pending
24 // `browse cloud fetch` processes concurrently.
25 const execFileAsync = promisify (execFile);
26
27 const args = process.argv. slice ( 2 );
28
29 if (args. includes ( '--help' ) || args. includes ( '-h' )) {
30 console. error ( `Usage: cat urls.txt | node gate_candidates.mjs [options]
31
32 Reads URLs from stdin (one per line) OR from --input <file>. For each URL, fetches
33 the homepage via \` browse cloud fetch --allow-redirects \` , extracts the first N chars of visible
34 text (the hero / tagline area), and classifies against include/exclude keyword rules.
35
36 Options:
37 --include "<csv>" Required. Comma-separated keywords; candidate PASSES if any match.
38 --exclude "<csv>" Comma-separated keywords; candidate REJECTS if any match.
39 --input <file> Read URLs from file instead of stdin.
40 --concurrency <n> Max parallel fetches (default: 6).
41 --hero-chars <n> Chars of visible text to examine (default: 800).
42 --help, -h Show this help message.` );
43 process. exit (args. includes ( '--help' ) || args. includes ( '-h' ) ? 0 : 1 );
44 }
45
46 function flag ( name ) {
47 const i = args. indexOf (name);
48 return i !== - 1 ? args[i + 1 ] : null ;
49 }
50
51 const includes = ( flag ( '--include' ) || '' ). split ( ',' ). map ( s => s. trim (). toLowerCase ()). filter (Boolean);
52 const excludes = ( flag ( '--exclude' ) || '' ). split ( ',' ). map ( s => s. trim (). toLowerCase ()). filter (Boolean);
53 // Floor at 1: `--concurrency 0` or a non-numeric value makes parseInt yield 0/NaN, which would
54 // spawn zero workers — the script would exit "successfully" having gated nothing, making
55 // discovery look empty with no error. Always run at least one worker.
56 const concurrency = Math. max ( 1 , parseInt ( flag ( '--concurrency' ) || '6' , 10 ) || 0 );
57 const heroChars = parseInt ( flag ( '--hero-chars' ) || '800' , 10 );
58 const inputFile = flag ( '--input' );
59
60 if (includes. length === 0 ) {
61 console. error ( 'Error: --include is required' );
62 process. exit ( 1 );
63 }
64
65 let urls;
66 if (inputFile) {
67 urls = readFileSync (inputFile, 'utf-8' ). split ( ' \n ' ). map ( l => l. trim ()). filter (Boolean);
68 } else {
69 const stdin = readFileSync ( 0 , 'utf-8' );
70 urls = stdin. split ( ' \n ' ). map ( l => l. trim ()). filter (Boolean);
71 }
72
73 if (urls. length === 0 ) {
74 console. error ( 'Error: no URLs provided (pipe via stdin or use --input)' );
75 process. exit ( 1 );
76 }
77
78 function stripHtml ( html ) {
79 return html
80 . replace ( /<script [ ^ >] * > [\s\S] *? < \/ script>/ gi , ' ' )
81 . replace ( /<style [ ^ >] * > [\s\S] *? < \/ style>/ gi , ' ' )
82 . replace ( /< [ ^ >] * >/ g , ' ' )
83 . replace ( /&/ g , '&' )
84 . replace ( /</ g , '<' )
85 . replace ( />/ g , '>' )
86 . replace ( /"/ g , '"' )
87 . replace ( /'/ g , "'" )
88 . replace ( / / g , ' ' )
89 . replace ( / \s + / g , ' ' )
90 . trim ();
91 }
92
93 // Position-aware classification:
94 // 1. Exclude term in <title> → REJECT (their primary identity is the excluded category)
95 // 2. Include term in <title> → PASS (their primary identity matches)
96 // 3. Include in early hero (200ch) → PASS iff no exclude in early hero
97 // 4. Otherwise → REJECT (default conservative)
98 // Rationale: <title> is the single strongest signal of what a company sells.
99 // Mid/late hero mentions (e.g. "we also support web scraping use cases") shouldn't
100 // disqualify a real competitor that self-identifies in its title as a cloud browser.
101 function classify ( title , heroFull , includes , excludes ) {
102 const titleLower = (title || '' ). toLowerCase ();
103 const heroLower = heroFull. toLowerCase ();
104 const heroEarly = heroLower. slice ( 0 , 200 );
105
106 const incTitle = includes. filter ( k => titleLower. includes (k));
107 const excTitle = excludes. filter ( k => titleLower. includes (k));
108 const incEarly = includes. filter ( k => heroEarly. includes (k));
109 const excEarly = excludes. filter ( k => heroEarly. includes (k));
110 const incHero = includes. filter ( k => heroLower. includes (k));
111 const excHero = excludes. filter ( k => heroLower. includes (k));
112
113 let status, reason;
114 if (incTitle. length > 0 && excTitle. length > 0 ) {
115 // Hybrid-identity title (e.g. "Browser Automation & Web Scraping API").
116 // Break the tie by the early hero — whichever category has more mentions wins.
117 if (incEarly. length > excEarly. length ) { status = 'PASS' ; reason = `title-hybrid→hero200 leans include(${ incEarly [ 0 ] || incTitle [ 0 ] })` ; }
118 else if (excEarly. length > incEarly. length ) { status = 'REJECT' ; reason = `title-hybrid→hero200 leans exclude(${ excEarly [ 0 ] || excTitle [ 0 ] })` ; }
119 else { status = 'PASS' ; reason = `title-hybrid→tie, defaulting include(${ incTitle [ 0 ] })` ; }
120 }
121 else if (excTitle. length > 0 ) { status = 'REJECT' ; reason = `title→exclude(${ excTitle [ 0 ] })` ; }
122 else if (incTitle. length > 0 ) { status = 'PASS' ; reason = `title→include(${ incTitle [ 0 ] })` ; }
123 else if (incEarly. length > 0 && excEarly. length === 0 ) { status = 'PASS' ; reason = `hero200→include(${ incEarly [ 0 ] })` ; }
124 else if (excEarly. length > 0 ) { status = 'REJECT' ; reason = `hero200→exclude(${ excEarly [ 0 ] })` ; }
125 else if (incHero. length > 0 && excHero. length === 0 ) { status = 'PASS' ; reason = `hero→include(${ incHero [ 0 ] })` ; }
126 // Late-hero conflict: both include AND exclude appear in chars 200–800 (nothing in
127 // title or early hero). This is genuine ambiguous signal, not absence — return UNKNOWN
128 // so the candidate surfaces in the user-confirmation bucket at Step 4.5 instead of
129 // being silently dropped as REJECT.
130 else if (incHero. length > 0 && excHero. length > 0 ) { status = 'UNKNOWN' ; reason = `hero→conflict(include:${ incHero [ 0 ] }, exclude:${ excHero [ 0 ] })` ; }
131 else { status = 'REJECT' ; reason = 'no category signal' ; }
132
133 return {
134 status, reason,
135 matched_includes: [ ...new Set ([ ... incTitle, ... incEarly, ... incHero])],
136 matched_excludes: [ ...new Set ([ ... excTitle, ... excEarly, ... excHero])],
137 };
138 }
139
140 async function gateOne ( url ) {
141 let stdout;
142 try {
143 // --format raw returns the JSON envelope with raw HTML in `.content` (the default
144 // is markdown, which has no <title> tag for the position-aware classifier to read).
145 const r = await execFileAsync ( 'browse' , [ 'cloud' , 'fetch' , '--allow-redirects' , '--format' , 'raw' , url], {
146 maxBuffer: 4 * 1024 * 1024 ,
147 timeout: 20000 ,
148 });
149 stdout = r.stdout;
150 } catch (err) {
151 // Non-zero exit, timeout, or spawn failure all surface here.
152 return { url, status: 'UNKNOWN' , reason: `browse cloud fetch failed: ${ err . message }` , matched_includes: [], matched_excludes: [], title: '' , hero: '' };
153 }
154 let resp;
155 try { resp = JSON . parse (stdout); } catch {
156 return { url, status: 'UNKNOWN' , reason: 'non-JSON response' , matched_includes: [], matched_excludes: [], title: '' , hero: '' };
157 }
158 const html = resp.content || '' ;
159 const titleM = html. match ( /<title [ ^ >] * >( [ ^ <] * )< \/ title>/ i );
160 const title = titleM ? titleM[ 1 ]. trim () : '' ;
161 const heroFull = stripHtml (html). slice ( 0 , heroChars);
162 const c = classify (title, heroFull, includes, excludes);
163 return {
164 url,
165 status: c.status,
166 reason: c.reason,
167 matched_includes: c.matched_includes,
168 matched_excludes: c.matched_excludes,
169 title,
170 hero: heroFull. slice ( 0 , 240 ),
171 };
172 }
173
174 // Run with bounded concurrency
175 const results = [];
176 async function runAll () {
177 const queue = [ ... urls];
178 const workers = Array (Math. min (concurrency, queue. length )). fill ( 0 ). map ( async () => {
179 while (queue. length > 0 ) {
180 const u = queue. shift ();
181 const r = await gateOne (u);
182 results. push (r);
183 console. log ( JSON . stringify (r));
184 }
185 });
186 await Promise . all (workers);
187 }
188
189 await runAll ();
190
191 const pass = results. filter ( r => r.status === 'PASS' ). length ;
192 const reject = results. filter ( r => r.status === 'REJECT' ). length ;
193 const unknown = results. filter ( r => r.status === 'UNKNOWN' ). length ;
194 console. error ( ` \n Gate: ${ pass } PASS / ${ reject } REJECT / ${ unknown } UNKNOWN (of ${ results . length })` );