Setting the file. One moment.
Bench Db · Sandbox Bench · vercel/next.js · Skills Docs
Back to the top of the page 24.4
Bench Common
const copy
— line 427
This file
Number 24.5
Position 5 of 12
Type JavaScript
Size 17 KB
Lines 574 scripts/ bench-db.mjs
JavaScript · 574 lines · 17 KB
14
import
path
from
'node:path'
15 import zlib from 'node:zlib'
16 import crypto from 'node:crypto'
17 import { execFileSync } from 'node:child_process'
18 import { DatabaseSync } from 'node:sqlite'
19 import { loadConfig } from './config.mjs'
20
21 const SCHEMA = `
22 PRAGMA journal_mode=WAL;
23 PRAGMA foreign_keys=ON;
24 CREATE TABLE IF NOT EXISTS runs(
25 run_id TEXT PRIMARY KEY,
26 label TEXT,
27 kind TEXT,
28 started_at TEXT,
29 imported_at TEXT NOT NULL,
30 harness_sha TEXT,
31 meta TEXT
32 );
33 CREATE TABLE IF NOT EXISTS samples(
34 sample_id INTEGER PRIMARY KEY,
35 run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE,
36 boot INTEGER NOT NULL,
37 arm TEXT NOT NULL,
38 block INTEGER NOT NULL,
39 run INTEGER NOT NULL,
40 route TEXT NOT NULL,
41 phase TEXT NOT NULL,
42 fingerprint TEXT,
43 version TEXT,
44 cpu TEXT,
45 errors INTEGER NOT NULL DEFAULT 0,
46 UNIQUE(run_id, boot, arm, block, run, route, phase)
47 );
48 CREATE TABLE IF NOT EXISTS measurements(
49 sample_id INTEGER NOT NULL REFERENCES samples(sample_id) ON DELETE CASCADE,
50 metric TEXT NOT NULL,
51 value REAL NOT NULL,
52 UNIQUE(sample_id, metric)
53 );
54 CREATE TABLE IF NOT EXISTS artifacts(
55 artifact_id INTEGER PRIMARY KEY,
56 run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE,
57 boot INTEGER,
58 arm TEXT,
59 name TEXT NOT NULL,
60 kind TEXT,
61 sha256 TEXT NOT NULL,
62 bytes INTEGER NOT NULL,
63 data BLOB NOT NULL,
64 UNIQUE(run_id, boot, arm, name)
65 );
66 CREATE TABLE IF NOT EXISTS boots(
67 run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE,
68 boot INTEGER NOT NULL,
69 vm_name TEXT,
70 UNIQUE(run_id, boot)
71 );
72 CREATE TABLE IF NOT EXISTS source_files(
73 run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE,
74 name TEXT NOT NULL,
75 sha256 TEXT NOT NULL,
76 rows INTEGER NOT NULL,
77 UNIQUE(run_id, name)
78 );
79 CREATE INDEX IF NOT EXISTS idx_samples_run ON samples(run_id);
80 CREATE INDEX IF NOT EXISTS idx_measurements_sample ON measurements(sample_id);
81 CREATE INDEX IF NOT EXISTS idx_artifacts_run ON artifacts(run_id);
82 `
83
84 // Row fields that identify a sample; every OTHER numeric field in a
85 // JSONL row is a measurement. New metrics therefore need no schema or
86 // importer change.
87 const IDENT = new Set ([
88 'vm' ,
89 'arm' ,
90 'block' ,
91 'run' ,
92 'route' ,
93 'phase' ,
94 'fp' ,
95 'ver' ,
96 'cpu' ,
97 'errors' ,
98 'payload' ,
99 'round' ,
100 ])
101
102 export function dbPath () {
103 return path. join ( loadConfig ({ requireScope: false }).cacheDir, 'results.db' )
104 }
105
106 export function openDb ( file = dbPath ()) {
107 fs. mkdirSync (path. dirname (file), { recursive: true })
108 const db = new DatabaseSync (file)
109 db. exec ( SCHEMA )
110 return db
111 }
112
113 function harnessSha () {
114 try {
115 return execFileSync (
116 'git' ,
117 [
118 '-C' ,
119 path. dirname ( new URL ( import . meta .url).pathname),
120 'rev-parse' ,
121 'HEAD' ,
122 ],
123 { encoding: 'utf8' }
124 ). trim ()
125 } catch {
126 return null
127 }
128 }
129
130 function artifactKind ( name ) {
131 if (name. endsWith ( '.cpuprofile' )) return 'cpuprofile'
132 if (name. endsWith ( '.json' )) return 'json'
133 if (name. endsWith ( '.log' ) || name. endsWith ( '.txt' )) return 'log'
134 return 'file'
135 }
136
137 function walk ( dir ) {
138 const out = []
139 for ( const e of fs. readdirSync (dir, { withFileTypes: true })) {
140 const p = path. join (dir, e.name)
141 if (e. isDirectory ()) out. push ( ... walk (p))
142 else if (e. isFile ()) out. push (p)
143 }
144 return out
145 }
146
147 // Import one run dir: replaces the run's rows wholesale so re-imports
148 // (recovery, added boots, new artifacts) converge on the same state.
149 // A re-import that would SHRINK a run (fewer boots or samples than the
150 // db already holds — e.g. a cleaned-up run dir) is refused without
151 // force: claims must never silently lose data underneath them.
152 export function importRun ( db , dir , { force = false } = {}) {
153 const runId = path. basename (path. resolve (dir))
154 const files = fs
155 . readdirSync (dir)
156 . filter (( f ) => / ^ results-vm \d + \. jsonl $ / . test (f))
157 . sort ()
158 . map (( f ) => path. join (dir, f))
159 if (files. length === 0 ) throw new Error ( `no results-vm*.jsonl in ${ dir }` )
160 const readJson = ( f ) => {
161 try {
162 return JSON . parse (fs. readFileSync (path. join (dir, f), 'utf8' ))
163 } catch {
164 return null
165 }
166 }
167 const meta = readJson ( 'meta.json' )
168 const status = readJson ( 'status.json' )
169 const prev = db
170 . prepare (
171 'SELECT COUNT(DISTINCT boot) boots, COUNT(*) n FROM samples WHERE run_id = ?'
172 )
173 . get (runId)
174 db. exec ( 'BEGIN' )
175 try {
176 db. prepare ( 'DELETE FROM runs WHERE run_id = ?' ). run (runId)
177 let kind = 'e2e'
178 const insSample = db. prepare ( `INSERT INTO samples
179 (run_id, boot, arm, block, run, route, phase, fingerprint, version, cpu, errors)
180 VALUES (?,?,?,?,?,?,?,?,?,?,?)` )
181 const insMeas = db. prepare (
182 'INSERT INTO measurements (sample_id, metric, value) VALUES (?,?,?)'
183 )
184 let samples = 0
185 let pendingRows = []
186 const fileInfos = []
187 for ( const [ boot , file ] of files. entries ()) {
188 const content = fs. readFileSync (file, 'utf8' )
189 let fileRows = 0
190 for ( const line of content. trim (). split ( ' \n ' )) {
191 if ( ! line) continue
192 const row = JSON . parse (line)
193 const micro = row.payload !== undefined
194 if (micro) kind = 'micro'
195 pendingRows. push ({ boot, row, micro })
196 fileRows ++
197 }
198 fileInfos. push ({
199 boot,
200 name: path. basename (file),
201 sha256: crypto. createHash ( 'sha256' ). update (content). digest ( 'hex' ),
202 rows: fileRows,
203 })
204 }
205 if (
206 ! force &&
207 prev.n > 0 &&
208 (files. length < prev.boots || pendingRows. length < prev.n)
209 ) {
210 throw new Error (
211 `re-import of ${ runId } would shrink it ` +
212 `(${ prev . boots } boots/${ prev . n } samples in db, ` +
213 `${ files . length } boots/${ pendingRows . length } samples in ${ dir }) — ` +
214 'pass --force only if the db copy is known bad'
215 )
216 }
217 db. prepare (
218 `INSERT INTO runs
219 (run_id, label, kind, started_at, imported_at, harness_sha, meta)
220 VALUES (?,?,?,?,?,?,?)`
221 ). run (
222 runId,
223 meta?.label ?? runId. replace ( / ^ run-/ , '' ). replace ( /- [a-z0-9] +$ / , '' ),
224 kind,
225 status?.startedAt ?? null ,
226 new Date (). toISOString (),
227 harnessSha (),
228 meta ? JSON . stringify (meta) : null
229 )
230 // Provenance: which sandbox VM produced each boot (from
231 // status.json, matched by the index in the VM name), and the exact
232 // bytes each boot's JSONL contributed.
233 const vmByIndex = new Map ()
234 for ( const name of Object. keys (status?.vms ?? {})) {
235 const idx = name. match ( /-( \d + )- [a-z0-9] +$ / )?.[ 1 ]
236 if (idx !== undefined ) vmByIndex. set ( Number (idx), name)
237 }
238 const insBoot = db. prepare (
239 'INSERT INTO boots (run_id, boot, vm_name) VALUES (?,?,?)'
240 )
241 const insFile = db. prepare (
242 'INSERT INTO source_files (run_id, name, sha256, rows) VALUES (?,?,?,?)'
243 )
244 for ( const f of fileInfos) {
245 const n = Number (f.name. match ( / ^ results-vm( \d + ) \. jsonl $ / )?.[ 1 ])
246 insBoot. run (runId, f.boot, vmByIndex. get (n) ?? null )
247 insFile. run (runId, f.name, f.sha256, f.rows)
248 }
249 for ( const { boot , row , micro } of pendingRows) {
250 const { lastInsertRowid : sid } = insSample. run (
251 runId,
252 boot,
253 row.arm,
254 micro ? 0 : (row.block ?? 0 ),
255 micro ? (row.round ?? 0 ) : (row.run ?? 0 ),
256 micro ? '' : (row.route ?? '' ),
257 micro ? row.payload : row.phase,
258 row.fp ?? null ,
259 row.ver ?? null ,
260 row.cpu ?? null ,
261 row.errors ?? 0
262 )
263 samples ++
264 for ( const [ k , v ] of Object. entries (row)) {
265 if ( IDENT . has (k)) continue
266 if ( typeof v !== 'number' || ! Number. isFinite (v)) continue
267 insMeas. run (sid, k, v)
268 }
269 }
270 // Artifacts: everything under prof-vm<N>/ (CPU profiles etc.),
271 // stored gzipped with the sha256 of the RAW content.
272 const insArt = db. prepare ( `INSERT INTO artifacts
273 (run_id, boot, arm, name, kind, sha256, bytes, data) VALUES (?,?,?,?,?,?,?,?)` )
274 let artifacts = 0
275 for ( const e of fs. readdirSync (dir)) {
276 const m = e. match ( / ^ prof-vm( \d + ) $ / )
277 if ( ! m) continue
278 const boot = Number (m[ 1 ])
279 for ( const f of walk (path. join (dir, e))) {
280 const rel = path. relative (path. join (dir, e), f)
281 const arm = rel. match ( / ^ prof-( [ ^ /] + ) \/ / )?.[ 1 ] ?? null
282 const raw = fs. readFileSync (f)
283 insArt. run (
284 runId,
285 boot,
286 arm,
287 rel,
288 artifactKind (rel),
289 crypto. createHash ( 'sha256' ). update (raw). digest ( 'hex' ),
290 raw. length ,
291 zlib. gzipSync (raw)
292 )
293 artifacts ++
294 }
295 }
296 db. exec ( 'COMMIT' )
297 return { runId, boots: files. length , samples, artifacts }
298 } catch (e) {
299 db. exec ( 'ROLLBACK' )
300 throw e
301 }
302 }
303
304 // Rows for bench-stats.analyzeE2eRows, reconstructed from the DB —
305 // the stats layer never reads run dirs.
306 export function loadRows ( db , runId ) {
307 const samples = db
308 . prepare ( 'SELECT * FROM samples WHERE run_id = ?' )
309 . all (runId)
310 if (samples. length === 0 ) throw new Error ( `no samples for run ${ runId }` )
311 const meas = db
312 . prepare (
313 `SELECT m.sample_id, m.metric, m.value FROM measurements m
314 JOIN samples s ON s.sample_id = m.sample_id WHERE s.run_id = ?`
315 )
316 . all (runId)
317 const byId = new Map ()
318 for ( const s of samples) {
319 byId. set (s.sample_id, {
320 vm: Number (s.boot),
321 arm: s.arm,
322 block: Number (s.block),
323 run: Number (s.run),
324 route: s.route,
325 phase: s.phase,
326 fp: s.fingerprint,
327 ver: s.version,
328 errors: Number (s.errors),
329 })
330 }
331 for ( const m of meas) byId. get (m.sample_id)[m.metric] = m.value
332 return [ ... byId. values ()]
333 }
334
335 export function runMeta ( db , runId ) {
336 const r = db. prepare ( 'SELECT * FROM runs WHERE run_id = ?' ). get (runId)
337 return r ? { ... r, meta: r.meta ? JSON . parse (r.meta) : null } : null
338 }
339
340 // Integrity checks. Everything here is mechanical: SQLite-level
341 // integrity, referential health, per-run shape (one fingerprint per
342 // arm, paired sample counts), and artifact hashes. Failures are
343 // data-integrity violations; notes are true facts a reader must not
344 // gloss over (e.g. a recovered run that is legitimately partial).
345 export function verify ( db , runId ) {
346 const problems = []
347 const notes = []
348 const ic = db. prepare ( 'PRAGMA integrity_check' ). all ()
349 if ( ! (ic. length === 1 && ic[ 0 ].integrity_check === 'ok' )) {
350 problems. push ( `sqlite integrity_check: ${ JSON . stringify ( ic ) }` )
351 }
352 const fk = db. prepare ( 'PRAGMA foreign_key_check' ). all ()
353 if (fk. length > 0 ) problems. push ( `foreign_key_check: ${ fk . length } violations` )
354 const runs = runId
355 ? db. prepare ( 'SELECT run_id FROM runs WHERE run_id = ?' ). all (runId)
356 : db. prepare ( 'SELECT run_id FROM runs' ). all ()
357 if (runId && runs. length === 0 ) problems. push ( `run ${ runId } not in db` )
358 for ( const { run_id } of runs) {
359 const orphanSamples = db
360 . prepare (
361 `SELECT COUNT(*) c FROM samples s
362 WHERE s.run_id = ? AND NOT EXISTS
363 (SELECT 1 FROM measurements m WHERE m.sample_id = s.sample_id)`
364 )
365 . get (run_id).c
366 if (orphanSamples > 0 ) {
367 problems. push ( `${ run_id }: ${ orphanSamples } samples with no measurements` )
368 }
369 const arms = db
370 . prepare (
371 `SELECT arm, COUNT(DISTINCT COALESCE(fingerprint,'') ||
372 '/' || COALESCE(version,'')) fps, COUNT(*) n
373 FROM samples WHERE run_id = ? GROUP BY arm`
374 )
375 . all (run_id)
376 if (arms. length !== 2 ) {
377 problems. push ( `${ run_id }: expected 2 arms, found ${ arms . length }` )
378 }
379 for ( const a of arms) {
380 if (a.fps > 1 ) {
381 problems. push (
382 `${ run_id }: arm ${ a . arm } has ${ a . fps } distinct fingerprints`
383 )
384 }
385 }
386 const planned = (() => {
387 try {
388 return JSON . parse (
389 db. prepare ( 'SELECT meta FROM runs WHERE run_id = ?' ). get (run_id).meta
390 ).vms
391 } catch {
392 return undefined
393 }
394 })()
395 const gotBoots = db
396 . prepare ( 'SELECT COUNT(DISTINCT boot) c FROM samples WHERE run_id = ?' )
397 . get (run_id).c
398 if (planned && gotBoots < planned) {
399 notes. push ( `${ run_id }: partial run (${ gotBoots }/${ planned } boots)` )
400 }
401 if (arms. length === 2 && arms[ 0 ].n !== arms[ 1 ].n) {
402 problems. push (
403 `${ run_id }: unpaired sample counts ` +
404 `(${ arms [ 0 ]. arm }=${ arms [ 0 ]. n } vs ${ arms [ 1 ]. arm }=${ arms [ 1 ]. n })`
405 )
406 }
407 for ( const art of db
408 . prepare (
409 'SELECT artifact_id, name, sha256, bytes, data FROM artifacts WHERE run_id = ?'
410 )
411 . all (run_id)) {
412 const raw = zlib. gunzipSync (art.data)
413 if (
414 raw. length !== Number (art.bytes) ||
415 crypto. createHash ( 'sha256' ). update (raw). digest ( 'hex' ) !== art.sha256
416 ) {
417 problems. push ( `${ run_id }: artifact ${ art . name } fails hash/size check` )
418 }
419 }
420 }
421 return { failures: problems, notes }
422 }
423
424 export function exportRuns ( db , outFile , runIds ) {
425 fs. rmSync (outFile, { force: true })
426 const out = openDb (outFile)
427 const copy = ( table , cols ) => {
428 const ins = out. prepare ( `INSERT INTO ${ table } (${ cols . join ( ',' ) })
429 VALUES (${ cols . map (() => '?' ). join ( ',' ) })` )
430 return ( row ) => ins. run ( ... cols. map (( c ) => row[c]))
431 }
432 out. exec ( 'BEGIN' )
433 const copyRun = copy ( 'runs' , [
434 'run_id' ,
435 'label' ,
436 'kind' ,
437 'started_at' ,
438 'imported_at' ,
439 'harness_sha' ,
440 'meta' ,
441 ])
442 const copySample = copy ( 'samples' , [
443 'sample_id' ,
444 'run_id' ,
445 'boot' ,
446 'arm' ,
447 'block' ,
448 'run' ,
449 'route' ,
450 'phase' ,
451 'fingerprint' ,
452 'version' ,
453 'cpu' ,
454 'errors' ,
455 ])
456 const copyMeas = copy ( 'measurements' , [ 'sample_id' , 'metric' , 'value' ])
457 const copyBoot = copy ( 'boots' , [ 'run_id' , 'boot' , 'vm_name' ])
458 const copyFile = copy ( 'source_files' , [ 'run_id' , 'name' , 'sha256' , 'rows' ])
459 const copyArt = copy ( 'artifacts' , [
460 'run_id' ,
461 'boot' ,
462 'arm' ,
463 'name' ,
464 'kind' ,
465 'sha256' ,
466 'bytes' ,
467 'data' ,
468 ])
469 for ( const runId of runIds) {
470 const run = db. prepare ( 'SELECT * FROM runs WHERE run_id = ?' ). get (runId)
471 if ( ! run) throw new Error ( `run ${ runId } not in db` )
472 copyRun (run)
473 for ( const s of db
474 . prepare ( 'SELECT * FROM samples WHERE run_id = ?' )
475 . all (runId)) {
476 copySample (s)
477 for ( const m of db
478 . prepare ( 'SELECT * FROM measurements WHERE sample_id = ?' )
479 . all (s.sample_id)) {
480 copyMeas (m)
481 }
482 }
483 for ( const a of db
484 . prepare ( 'SELECT * FROM artifacts WHERE run_id = ?' )
485 . all (runId)) {
486 copyArt (a)
487 }
488 for ( const b of db
489 . prepare ( 'SELECT * FROM boots WHERE run_id = ?' )
490 . all (runId)) {
491 copyBoot (b)
492 }
493 for ( const f of db
494 . prepare ( 'SELECT * FROM source_files WHERE run_id = ?' )
495 . all (runId)) {
496 copyFile (f)
497 }
498 }
499 out. exec ( 'COMMIT' )
500 out. close ()
501 }
502
503 // ------------------------------------------------------------------ CLI
504 function isMain () {
505 if ( ! process.argv[ 1 ]) return false
506 try {
507 return (
508 fs. realpathSync (process.argv[ 1 ]) ===
509 fs. realpathSync ( new URL ( import . meta .url).pathname)
510 )
511 } catch {
512 return false
513 }
514 }
515 if ( isMain ()) {
516 const [ cmd , ... rest ] = process.argv. slice ( 2 )
517 const db = openDb ()
518 const report = ({ failures , notes }, label ) => {
519 for ( const n of notes) console. log ( `note: ${ n }` )
520 if (failures. length ) {
521 console. error ( `VERIFY FAILED: \n ${ failures . join ( ' \n ' ) }` )
522 process. exit ( 1 )
523 }
524 console. log ( `verify: ok${ label ? ` (${ label })` : ''}` )
525 }
526 if (cmd === 'import' ) {
527 const force = rest. includes ( '--force' )
528 const dirs = rest. filter (( a ) => a !== '--force' )
529 if (dirs. length === 0 ) {
530 console. error ( 'usage: bench-db.mjs import [--force] <runDir...>' )
531 process. exit ( 1 )
532 }
533 for ( const dir of dirs) {
534 let r
535 try {
536 r = importRun (db, dir, { force })
537 } catch (e) {
538 console. error ( `IMPORT REFUSED: ${ e . message }` )
539 process. exit ( 1 )
540 }
541 console. log (
542 `${ r . runId }: ${ r . boots } boots, ${ r . samples } samples, ${ r . artifacts } artifacts`
543 )
544 }
545 report ( verify (db))
546 } else if (cmd === 'verify' ) {
547 report ( verify (db, rest[ 0 ]), rest[ 0 ] ?? 'all runs' )
548 } else if (cmd === 'export' ) {
549 const [ out , ... ids ] = rest
550 if ( ! out || ids. length === 0 ) {
551 console. error ( 'usage: bench-db.mjs export <out.db> <runId...>' )
552 process. exit ( 1 )
553 }
554 exportRuns (db, out, ids)
555 console. log ( `${ out }: ${ ids . length } runs` )
556 } else if (cmd === 'ls' ) {
557 for ( const r of db
558 . prepare (
559 `SELECT r.run_id, r.kind, r.started_at,
560 (SELECT COUNT(DISTINCT boot) FROM samples s WHERE s.run_id = r.run_id) boots,
561 (SELECT COUNT(*) FROM samples s WHERE s.run_id = r.run_id) samples,
562 (SELECT COUNT(*) FROM artifacts a WHERE a.run_id = r.run_id) artifacts
563 FROM runs r ORDER BY r.started_at`
564 )
565 . all ()) {
566 console. log (
567 `${ r . run_id } ${ r . kind } boots=${ r . boots } samples=${ r . samples } artifacts=${ r . artifacts } ${ r . started_at ?? ''}`
568 )
569 }
570 } else {
571 console. error ( 'usage: bench-db.mjs import|verify|export|ls' )
572 process. exit ( 1 )
573 }
574 }