Setting the file. One moment.
Extract Page · Company Research · browserbase/skills · Skills Docs
Back to the top of the page scripts/ extract_page.mjs
JavaScript · 180 lines · 5 KB
;
12 import { join } from "node:path" ;
13
14 const THIN_CONTENT_THRESHOLD = 200 ; // body chars under this → JS-rendered, fall back
15
16 function parseArgs ( argv ) {
17 const args = { url: null , maxChars: 3000 };
18 for ( let i = 0 ; i < argv. length ; i ++ ) {
19 const a = argv[i];
20 if (a === "--max-chars" ) args.maxChars = parseInt (argv[ ++ i], 10 );
21 else if ( ! args.url) args.url = a;
22 }
23 if ( ! args.url) {
24 console. error ( "Usage: extract_page.mjs <url> [--max-chars N]" );
25 process. exit ( 2 );
26 }
27 return args;
28 }
29
30 function browseFetch ( url , outFile ) {
31 execFileSync ( "browse" , [ "cloud" , "fetch" , "--allow-redirects" , url, "--output" , outFile], {
32 stdio: [ "ignore" , "ignore" , "ignore" ],
33 });
34 }
35
36 function browseGetMarkdown ( url ) {
37 const session = `extract-page-${ process . pid }-${ Date . now () }` ;
38 const env = { ... process.env, BROWSE_SESSION: session };
39 try {
40 execFileSync ( "browse" , [ "open" , url, "--local" , "--headless" ], {
41 stdio: [ "ignore" , "ignore" , "ignore" ],
42 timeout: 90000 ,
43 env,
44 });
45 const out = execFileSync ( "browse" , [ "get" , "markdown" ], {
46 encoding: "utf8" ,
47 timeout: 90000 ,
48 maxBuffer: 50 * 1024 * 1024 ,
49 env,
50 });
51 // browse prints banners (e.g. "Update available...") before the JSON blob.
52 // Find the first '{' and try to JSON.parse from there.
53 const start = out. indexOf ( "{" );
54 if (start < 0 ) return "" ;
55 try {
56 const parsed = JSON . parse (out. slice (start));
57 if (parsed && typeof parsed.markdown === "string" ) return parsed.markdown;
58 } catch {
59 // Fallback: extract "markdown": "..." with a lenient regex that handles
60 // escaped quotes and newlines.
61 const m = out. slice (start). match ( /"markdown" \s * : \s * "((?: \\ . | [ ^ " \\ ] ) * )"/ s );
62 if (m) {
63 try { return JSON . parse ( `"${ m [ 1 ] }"` ); } catch { return m[ 1 ]; }
64 }
65 }
66 return "" ;
67 } catch (err) {
68 return "" ;
69 } finally {
70 try {
71 execFileSync ( "browse" , [ "stop" ], {
72 stdio: [ "ignore" , "ignore" , "ignore" ],
73 timeout: 15000 ,
74 env,
75 });
76 } catch {}
77 }
78 }
79
80 function extractMeta ( html , name , attr = "name" ) {
81 const re = new RegExp (
82 `<meta \\ s+${ attr }=["']${ name }["'] \\ s+content=["']([^"']*)["']` ,
83 "i"
84 );
85 const re2 = new RegExp (
86 `<meta \\ s+content=["']([^"']*)["'] \\ s+${ attr }=["']${ name }["']` ,
87 "i"
88 );
89 const m = html. match (re) || html. match (re2);
90 return m ? m[ 1 ]. trim () : "" ;
91 }
92
93 function extractTitle ( html ) {
94 const m = html. match ( /<title [ ^ >] * >( [ ^ <] * )< \/ title>/ i );
95 return m ? m[ 1 ]. trim () : "" ;
96 }
97
98 function extractVisibleText ( html , maxChars ) {
99 // Multi-line aware script/style removal.
100 let s = html
101 . replace ( /<script \b [ ^ >] * > [\s\S] *? < \/ script>/ gi , " " )
102 . replace ( /<style \b [ ^ >] * > [\s\S] *? < \/ style>/ gi , " " )
103 . replace ( /<noscript \b [ ^ >] * > [\s\S] *? < \/ noscript>/ gi , " " )
104 . replace ( /<!-- [\s\S] *? -->/ g , " " )
105 . replace ( /< [ ^ >] + >/ g , " " )
106 . replace ( / / g , " " )
107 . replace ( /&/ g , "&" )
108 . replace ( /</ g , "<" )
109 . replace ( />/ g , ">" )
110 . replace ( /"/ g , '"' )
111 . replace ( /'/ g , "'" )
112 . replace ( /&# [0-9] + ;/ g , " " )
113 . replace ( / \s + / g , " " )
114 . trim ();
115 return s. slice ( 0 , maxChars);
116 }
117
118 function extractHeadings ( html , limit = 10 ) {
119 const re = /<h [1-3][ ^ >] * >( [\s\S] *? )< \/ h [1-3] >/ gi ;
120 const out = [];
121 let m;
122 while ((m = re. exec (html)) && out. length < limit) {
123 const text = m[ 1 ]. replace ( /< [ ^ >] + >/ g , "" ). replace ( / \s + / g , " " ). trim ();
124 if (text) out. push (text);
125 }
126 return out;
127 }
128
129 function main () {
130 const { url , maxChars } = parseArgs (process.argv. slice ( 2 ));
131 const dir = mkdtempSync ( join ( tmpdir (), "extract_page_" ));
132 const htmlFile = join (dir, "page.html" );
133
134 let html = "" ;
135 let fetchOk = false ;
136 try {
137 browseFetch (url, htmlFile);
138 html = readFileSync (htmlFile, "utf8" );
139 fetchOk = true ;
140 } catch (err) {
141 console. error ( `[extract_page] browse cloud fetch failed: ${ err . message }` );
142 }
143
144 const title = extractTitle (html);
145 const metaDesc = extractMeta (html, "description" );
146 const ogTitle = extractMeta (html, "og:title" , "property" );
147 const ogDesc = extractMeta (html, "og:description" , "property" );
148 const headings = extractHeadings (html);
149 let body = extractVisibleText (html, maxChars);
150
151 // Thin content → JS-rendered SPA → fall back to browse get markdown.
152 let fallbackUsed = false ;
153 if (body. length < THIN_CONTENT_THRESHOLD ) {
154 const md = browseGetMarkdown (url);
155 if (md && md. length > body. length ) {
156 body = md. replace ( / \s + / g , " " ). slice ( 0 , maxChars);
157 fallbackUsed = true ;
158 }
159 }
160
161 rmSync (dir, { recursive: true , force: true });
162
163 // Structured output for subagent to read.
164 const lines = [
165 `URL: ${ url }` ,
166 `FETCH_OK: ${ fetchOk }` ,
167 `FALLBACK_TO_BROWSE: ${ fallbackUsed }` ,
168 `TITLE: ${ title }` ,
169 `META_DESCRIPTION: ${ metaDesc }` ,
170 `OG_TITLE: ${ ogTitle }` ,
171 `OG_DESCRIPTION: ${ ogDesc }` ,
172 `HEADINGS: ${ headings . join ( " | " ) }` ,
173 `BODY_CHARS: ${ body . length }` ,
174 `BODY:` ,
175 body,
176 ];
177 process.stdout. write (lines. join ( " \n " ) + " \n " );
178 }
179
180 main ();