Setting the file. One moment.
Load · Browser To API · browserbase/skills · Skills Docs
Back to the top of the page Script
scripts/ load.mjs
JavaScript · 172 lines · 6 KB
'node:fs'
;
15 import path from 'node:path' ;
16 import { readJsonl, writeJsonl, intermediatePath, ensureDir } from './lib/io.mjs' ;
17
18 const KEEP_TYPES = new Set ([ 'XHR' , 'Fetch' , 'Document' ]);
19
20 function tryParseJson ( s ) {
21 if ( typeof s !== 'string' ) return s;
22 try { return JSON . parse (s); } catch { return s; }
23 }
24
25 function looksApiUrl ( url ) {
26 return / \/ (api | graphql | rest | v \d + ) \b / i . test (url) ||
27 / \. (json | jsonl | ndjson)( \? |$ )/ i . test (url);
28 }
29
30 function urlPath ( u ) {
31 try { return new URL (u).pathname; } catch { return u; }
32 }
33
34 function urlOrigin ( u ) {
35 try { const x = new URL (u); return `${ x . protocol }//${ x . host }` ; } catch { return null ; }
36 }
37
38 function urlQuery ( u ) {
39 try {
40 const x = new URL (u);
41 const out = {};
42 // First value wins for repeats. The downstream consumer (normalize.mjs)
43 // only uses parameter names + a representative value for type inference,
44 // so collapsing repeats to the first observation is fine.
45 for ( const [ k , v ] of x.searchParams. entries ()) {
46 if (out[k] === undefined ) out[k] = v;
47 }
48 return out;
49 } catch { return {}; }
50 }
51
52 // Walk a `browse network` capture directory and return a Map keyed by the
53 // CDP requestId, each value `{ reqBody, respBody }`. Bodies that are valid JSON
54 // are returned parsed; otherwise the raw string is preserved.
55 function loadBrowseNetworkBodies ( bodiesDir ) {
56 const out = new Map ();
57 if ( ! bodiesDir || ! fs. existsSync (bodiesDir)) return out;
58 const entries = fs. readdirSync (bodiesDir, { withFileTypes: true });
59 for ( const e of entries) {
60 if ( ! e. isDirectory ()) continue ;
61 const subdir = path. join (bodiesDir, e.name);
62 const reqPath = path. join (subdir, 'request.json' );
63 const respPath = path. join (subdir, 'response.json' );
64 if ( ! fs. existsSync (reqPath)) continue ;
65 let req, resp;
66 try { req = JSON . parse (fs. readFileSync (reqPath, 'utf8' )); } catch { continue ; }
67 try { resp = fs. existsSync (respPath) ? JSON . parse (fs. readFileSync (respPath, 'utf8' )) : null ; } catch { resp = null ; }
68 if ( ! req?.id) continue ;
69 const reqBody = req.body != null ? tryParseJson (req.body) : null ;
70 const respBody = resp?.body != null ? tryParseJson (resp.body) : null ;
71 out. set ( String (req.id), { reqBody, respBody });
72 }
73 return out;
74 }
75
76 export function load ( runPath , outDir , opts = {}) {
77 const cdpDir = path. join (runPath, 'cdp' , 'network' );
78 const requests = readJsonl (path. join (cdpDir, 'requests.jsonl' ));
79 const responses = readJsonl (path. join (cdpDir, 'responses.jsonl' ));
80
81 // Body sources: explicit --bodies path > <run>/cdp/network/bodies/ if present
82 let bodiesDir = opts.bodies || null ;
83 if ( ! bodiesDir) {
84 const stashed = path. join (runPath, 'cdp' , 'network' , 'bodies' );
85 if (fs. existsSync (stashed)) bodiesDir = stashed;
86 }
87 const bodyMap = loadBrowseNetworkBodies (bodiesDir);
88
89 // Index responses by requestId; if the trace has duplicates (redirects), the
90 // last one wins so the terminal status code is what we keep.
91 const respByReq = new Map ();
92 for ( const ev of responses) {
93 const rid = ev?.params?.requestId;
94 if (rid) respByReq. set (rid, ev);
95 }
96
97 const paired = [];
98 for ( const ev of requests) {
99 const p = ev?.params;
100 if ( ! p?.request) continue ;
101
102 const method = p.request.method;
103 const url = p.request.url;
104 if ( ! url || ! method) continue ;
105 if (method === 'OPTIONS' ) continue ;
106 if (url. startsWith ( 'data:' ) || url. startsWith ( 'blob:' )) continue ;
107
108 // Resource type: prefer p.type (CDP), fall back to URL heuristic.
109 const type = p.type || 'Other' ;
110 if ( ! KEEP_TYPES . has (type) && ! looksApiUrl (url)) continue ;
111
112 const respEv = respByReq. get (p.requestId);
113 const resp = respEv?.params?.response;
114 const status = resp?.status ?? null ;
115 if (status && status >= 300 && status < 400 ) {
116 // Pure redirect. The browser will issue a follow-up request with the
117 // same requestId carrying redirectResponse on it; we already record the
118 // post-redirect resource via the next requestWillBeSent. Drop the
119 // intermediate.
120 continue ;
121 }
122
123 const contentType = resp?.headers
124 ? Object. entries (resp.headers). find (([ k ]) => k. toLowerCase () === 'content-type' )?.[ 1 ] ?? null
125 : null ;
126
127 let reqBody = p.request.postData ? tryParseJson (p.request.postData) : null ;
128 let respBody = null ;
129
130 // Augment with browse-network bodies when present. Match by requestId
131 // (the browse-network entry's `id` IS the CDP requestId for XHR/Fetch).
132 const captured = bodyMap. get ( String (p.requestId));
133 if (captured) {
134 if (reqBody == null && captured.reqBody != null ) reqBody = captured.reqBody;
135 if (captured.respBody != null ) respBody = captured.respBody;
136 }
137
138 paired. push ({
139 requestId: p.requestId,
140 method,
141 url,
142 origin: urlOrigin (url),
143 path: urlPath (url),
144 query: urlQuery (url),
145 status,
146 type,
147 contentType,
148 reqHeaders: p.request.headers || {},
149 reqBody,
150 respHeaders: resp?.headers || {},
151 respBody,
152 ts: typeof p.wallTime === 'number' ? Math. round (p.wallTime * 1000 ) : null ,
153 });
154 }
155
156 ensureDir (path. join (outDir, 'intermediate' ));
157 writeJsonl ( intermediatePath (outDir, 'paired.jsonl' ), paired);
158 return {
159 count: paired. length ,
160 requests: requests. length ,
161 responses: responses. length ,
162 bodiesAttached: paired. filter ( r => r.respBody != null ). length ,
163 bodiesDir,
164 };
165 }
166
167 if ( import . meta .url === `file://${ process . argv [ 1 ] }` ) {
168 const [ run , out , bodies ] = process.argv. slice ( 2 );
169 if ( ! run || ! out) { console. error ( 'usage: load.mjs <run-path> <out-dir> [bodies-dir]' ); process. exit ( 2 ); }
170 const stats = load (run, out, { bodies });
171 console. log ( `load: ${ stats . count } paired (from ${ stats . requests } req / ${ stats . responses } resp)${ stats . bodiesAttached ? `, ${ stats . bodiesAttached } response bodies attached` : ''}` );
172 }