Setting the file. One moment.
Sandbox E2e · Sandbox Bench · vercel/next.js · Skills Docs
Back to the top of the page 24.4
Bench Common
539
async function runVm
— line 539
This file
Number 24.9
Position 9 of 12
Type JavaScript
Size 36 KB
Lines 1,008 scripts/ sandbox-e2e.mjs
JavaScript · 1,008 lines · 36 KB
15 // node sandbox-e2e.mjs --next-arms base=<ref>,cand=<ref> [--react-ref main]
16 // Common: [--blocks 1] [--runs 2] [--vms 16] [--routes /blog,/dashboard,/docs]
17 // [--warmup 200] [--serial 800] [--load-requests 8] [--load-concurrency 8]
18 // [--isolate-routes] [--bench-env K=V] [--profile] [--keep] [--dry-run]
19 import fs from 'fs'
20 import os from 'os'
21 import path from 'path'
22 import { analyzeE2eRows, tTestP } from './bench-stats.mjs'
23 import { openDb, importRun, loadRows, verify as verifyDb } from './bench-db.mjs'
24 import {
25 execFileP,
26 CONFIG,
27 NEXT_REPO_LAZY,
28 REACT_REPO_LAZY,
29 CACHE,
30 SETUP_VERSION,
31 REACT_GH_REPO,
32 NEXT_GH_REPO,
33 status,
34 writeStatus,
35 sb,
36 sbCpToVm,
37 sbExec,
38 rmVm,
39 runDetached,
40 resolvePrArms,
41 assertCiGreen,
42 commitTitle,
43 printRunContext,
44 makeLive,
45 sha256,
46 snapshotIdFor,
47 takeSnapshot,
48 ensureRefArm,
49 } from './bench-common.mjs'
50
51 // Runtime provenance: the compiled server files prod app-page runtimes
52 // are bundled from — BOTH bundlers, so changes touching only one still
53 // move the fingerprint. One file per server-side React layer (Flight,
54 // Fizz, shared react-server runtime): a change confined to one layer
55 // leaves the other layers' files byte-identical.
56 const FP_FILES = [
57 'packages/next/dist/compiled/react-server-dom-turbopack-experimental/cjs/react-server-dom-turbopack-server.node.production.js' ,
58 'packages/next/dist/compiled/react-server-dom-webpack-experimental/cjs/react-server-dom-webpack-server.node.production.js' ,
59 'packages/next/dist/compiled/react-dom-experimental/cjs/react-dom-server.node.production.js' ,
60 'packages/next/dist/compiled/react-experimental/cjs/react.react-server.production.js' ,
61 ]
62
63 function parseArgs () {
64 const a = process.argv. slice ( 2 )
65 const get = ( name , dflt ) => {
66 const i = a. indexOf (name)
67 return i >= 0 ? a[i + 1 ] : dflt
68 }
69 // React arms: name=<git ref in the react repo> (built remotely, cached
70 // by sha). Next arms: name=<git ref in this checkout>. Exactly one of
71 // the two sides varies; the other is fixed for both arms.
72 const parseArmSpec = ( spec , flag ) =>
73 spec
74 . split ( ',' )
75 . filter (Boolean)
76 . map (( s ) => {
77 const [ name , src ] = s. split ( '=' )
78 if ( ! name || ! src)
79 throw new Error ( `bad arm "${ s }" in ${ flag }, want name=<ref>` )
80 return { name, ref: src }
81 })
82 const arms = parseArmSpec ( get ( '--arms' , '' ), '--arms' )
83 const nextArms = parseArmSpec ( get ( '--next-arms' , '' ), '--next-arms' )
84 const pr = get ( '--pr' , undefined )
85 const nextPr = get ( '--next-pr' , undefined )
86 const reactModes = (arms. length ? 1 : 0 ) + (pr ? 1 : 0 )
87 const nextModes = (nextArms. length ? 1 : 0 ) + (nextPr ? 1 : 0 )
88 if (reactModes + nextModes !== 1 ) {
89 throw new Error ( 'need exactly one of: --pr, --arms, --next-pr, --next-arms' )
90 }
91 // One react arm = "vs what this Next ships": base derived as
92 // merge-base(cand, react synced into the Next ref).
93 if (arms. length > 2 )
94 throw new Error (
95 '--arms takes one arm (vs synced react) or two (base first)'
96 )
97 if (nextArms. length && nextArms. length !== 2 )
98 throw new Error ( '--next-arms needs exactly two arms (base first)' )
99 return {
100 arms,
101 nextArms,
102 pr,
103 nextPr,
104 dryRun: a. includes ( '--dry-run' ),
105 allowUngated: a. includes ( '--allow-ungated' ),
106 // Fixed sides. Next defaults to canary.
107 nextRef: get ( '--next-ref' , 'canary' ),
108 // For Next A/B the React side defaults to whatever each Next ref
109 // vendors (that's what would ship); --react-ref pins both arms to
110 // one React build instead.
111 reactRef: get ( '--react-ref' , '' ),
112 // The VM boot is the unit of replication (see bench-stats.mjs):
113 // allocate toward more boots with fewer runs each.
114 blocks: Number ( get ( '--blocks' , '1' )),
115 runs: Number ( get ( '--runs' , '2' )),
116 vms: Number ( get ( '--vms' , '16' )),
117 routes: get ( '--routes' , '/blog,/dashboard,/docs' ),
118 warmup: Number ( get ( '--warmup' , '200' )),
119 serial: Number ( get ( '--serial' , '800' )),
120 loadRequests: Number ( get ( '--load-requests' , '8' )),
121 loadConcurrency: Number ( get ( '--load-concurrency' , '8' )),
122 isolateRoutes: a. includes ( '--isolate-routes' ),
123 keep: a. includes ( '--keep' ),
124 prepare: a. includes ( '--prepare' ),
125 // Profiles are captured by default: the pass runs strictly after the
126 // timed runs (never touches the numbers), costs ~10-15 min of VM
127 // wall-clock, and cross-VM profile diffs proved highly stable
128 // (16/16 sign agreement on movers). --no-profile opts out.
129 profile: ! a. includes ( '--no-profile' ),
130 // KEY=VALUE env exported around bench:render-pipeline (e.g.
131 // NEXT_FLIGHT_RENDER=0 to force the byte-tee SSR baseline).
132 benchEnv: get ( '--bench-env' , '' ),
133 label: get ( '--label' , 'e2e' ),
134 }
135 }
136
137 // Normalize every mode into two arms of {name, ref (react), nextRef};
138 // exactly one side differs between the arms.
139 // The react commit a Next ref ships: sync-react records it in the root
140 // package.json ("react-builtin": "npm:react@19.x.y-canary-<sha>-<date>").
141 async function syncedReactSha ( nextRef ) {
142 const pkg = (
143 await execFileP (
144 'git' ,
145 [ '-C' , NEXT_REPO_LAZY (), 'show' , `${ nextRef }:package.json` ],
146 { maxBuffer: 1 << 24 }
147 )
148 ).stdout
149 const m = JSON . parse (pkg).devDependencies?.[ 'react-builtin' ]?. match (
150 /-( [0-9a-f] {8,40} )- \d {8}$ /
151 )
152 if ( ! m)
153 throw new Error (
154 `cannot parse synced react sha from ${ nextRef }:package.json react-builtin`
155 )
156 return m[ 1 ]
157 }
158
159 // Next refs: shas and refs already fetched this run resolve locally;
160 // branch/tag names resolve against the remote. Each ref resolves once
161 // per run (memo) and everything downstream uses the sha, so the two
162 // arms always get the same tree. The clone is shared between
163 // concurrent launchers (pid-namespaced temp refs, fetch retried).
164 async function fetchNextRef ( repo , spec , dst ) {
165 for ( let attempt = 1 ; ; attempt ++ ) {
166 try {
167 await execFileP ( 'git' , [
168 '-C' ,
169 repo,
170 'fetch' ,
171 '-q' ,
172 CONFIG .nextRepoUrl,
173 `+${ spec }:${ dst }` ,
174 ])
175 return
176 } catch (e) {
177 if (attempt >= 3 ) throw e
178 await new Promise (( r ) => setTimeout (r, 2000 * attempt))
179 }
180 }
181 }
182 const nextShaMemo = new Map ()
183 async function nextShaFor ( ref ) {
184 if (nextShaMemo. has (ref)) return nextShaMemo. get (ref)
185 const repo = NEXT_REPO_LAZY ()
186 let sha
187 if ( / ^ [0-9a-f] {7,40}$ / i . test (ref) || ref. startsWith ( 'refs/' )) {
188 try {
189 sha = (
190 await execFileP ( 'git' , [
191 '-C' ,
192 repo,
193 'rev-parse' ,
194 '--verify' ,
195 `${ ref }^{commit}` ,
196 ])
197 ).stdout. trim ()
198 } catch {}
199 }
200 if ( ! sha) {
201 const dst = `refs/bench-tmp/${ process . pid }/next-fixed-${ nextShaMemo . size }`
202 // "canary" means the latest published canary release, not the
203 // branch head: postinstall downloads the @next/swc binary for
204 // package.json's version, which only exists once that release is
205 // on npm. Between releases this is also a stable sha, so built
206 // snapshots stay warm until a new canary actually ships.
207 let spec = ref
208 let release
209 if (ref === 'canary' ) {
210 release = `v${ ( await execFileP ( 'npm' , [ 'view' , 'next@canary' , 'version' ])). stdout . trim () }`
211 spec = `refs/tags/${ release }`
212 }
213 await fetchNextRef (repo, spec, dst)
214 sha = (
215 await execFileP ( 'git' , [ '-C' , repo, 'rev-parse' , `${ dst }^{commit}` ])
216 ).stdout. trim ()
217 console. error (
218 `next ${ ref }: ${ release ? `${ release } = ` : ''}${ sha . slice ( 0 , 12 ) }`
219 )
220 }
221 nextShaMemo. set (ref, sha)
222 return sha
223 }
224
225 async function resolveArms ( cfg ) {
226 let arms
227 if (cfg.pr) {
228 arms = (
229 await resolvePrArms (
230 cfg.pr,
231 REACT_REPO_LAZY (),
232 CONFIG .reactRepoUrl,
233 'main'
234 )
235 ). map (( a ) => ({ ... a, nextRef: cfg.nextRef }))
236 } else if (cfg.arms. length === 1 ) {
237 // Candidate react vs whatever this Next ref ships. merge-base keeps
238 // the base a real commit in the candidate's history even when the
239 // synced version isn't an exact ancestor.
240 const synced = await syncedReactSha ( await nextShaFor (cfg.nextRef))
241 const repo = REACT_REPO_LAZY ()
242 const base = (
243 await execFileP ( 'git' , [
244 '-C' ,
245 repo,
246 'merge-base' ,
247 synced,
248 cfg.arms[ 0 ].ref,
249 ])
250 ).stdout. trim ()
251 console. error (
252 `react base = merge-base(${ cfg . arms [ 0 ]. ref }, synced ${ synced }) = ${ base . slice ( 0 , 12 ) }`
253 )
254 arms = [
255 { name: 'synced' , ref: base, nextRef: cfg.nextRef },
256 { ... cfg.arms[ 0 ], nextRef: cfg.nextRef },
257 ]
258 } else if (cfg.arms. length ) {
259 arms = cfg.arms. map (( a ) => ({ ... a, nextRef: cfg.nextRef }))
260 } else if (cfg.nextPr) {
261 arms = (
262 await resolvePrArms (
263 cfg.nextPr,
264 NEXT_REPO_LAZY (),
265 CONFIG .nextRepoUrl,
266 'canary'
267 )
268 ). map (( a ) => ({ name: a.name, ref: cfg.reactRef || null , nextRef: a.ref }))
269 } else {
270 arms = cfg.nextArms. map (( a ) => ({
271 name: a.name,
272 ref: cfg.reactRef || null ,
273 nextRef: a.ref,
274 }))
275 }
276 for ( const arm of arms) {
277 arm.nextSha = await nextShaFor (arm.nextRef)
278 }
279 return arms
280 }
281
282 // Human context for reports: PR title/URL and the varying side's
283 // commit titles, recorded in meta.json and printed with the analysis
284 // so verdicts can link what was measured.
285 async function describeRun ( cfg ) {
286 const reactVaries = ! (cfg.nextPr || cfg.nextArms. length )
287 let pr
288 const num = String (cfg.pr ?? cfg.nextPr ?? '' ). match ( /( \d + ) \/ ?$ / )?.[ 1 ]
289 if (num) {
290 const ghRepo = cfg.pr ? REACT_GH_REPO : NEXT_GH_REPO
291 pr = { url: `https://github.com/${ ghRepo }/pull/${ num }` }
292 try {
293 pr. title = (
294 await execFileP ( 'gh' , [
295 'api' ,
296 `repos/${ ghRepo }/pulls/${ num }` ,
297 '--jq' ,
298 '.title' ,
299 ])
300 ).stdout. trim ()
301 } catch {}
302 }
303 const arms = []
304 for ( const a of cfg.arms) {
305 arms. push ({
306 name: a.name,
307 title: reactVaries
308 ? await commitTitle ( REACT_REPO_LAZY (), a.ref)
309 : await commitTitle ( NEXT_REPO_LAZY (), a.nextSha),
310 })
311 }
312 return { pr, arms }
313 }
314 // ------------------------------------------------------------ snapshots
315
316 // Experiment snapshot: both arms fully vendored + built as SEPARATE repo
317 // trees (/vercel/sandbox/next-<arm>), app .next included, so run VMs
318 // boot straight into measurement and arm switching is a cd. Keyed on
319 // (next ref, armA, armB); the app build happens once here, so every run
320 // VM measures byte-identical artifacts.
321 function armId ( a ) {
322 return `${ a . sha ? a . sha . slice ( 0 , 12 ) : 'vendored'}+${ a . nextSha . slice ( 0 , 12 ) }`
323 }
324
325 async function ensureExperimentSnapshot ( cfg ) {
326 // Arm NAMES are part of the key, not just shas: tree paths inside the
327 // snapshot embed the names (/vercel/sandbox/next-<name>), so a snapshot
328 // built for the same sha pair under different names has the wrong trees.
329 const key = await sha256 (
330 SETUP_VERSION + 'exp3' + cfg.arms. map (( a ) => `${ a . name }=${ armId ( a ) }` ). join ()
331 )
332 let id = await snapshotIdFor ( CACHE , key)
333 if (id) return id
334 // Built TREES are cached independently of the (pair, names) snapshot:
335 // the base side of a comparison repeats across cells far more often
336 // than the exact pair does, and a cached tree turns a ~15 min
337 // install+build+sync into an upload+extract.
338 for ( const arm of cfg.arms) {
339 arm.treeKey = await sha256 ( SETUP_VERSION + 'tree1' + armId (arm))
340 arm.treeTgz = path. join ( CACHE , `tree-${ arm . treeKey }.tgz` )
341 arm.treeCached = fs. existsSync (arm.treeTgz)
342 }
343 const vm = `sbench-expsnap-${ Date . now (). toString ( 36 ) }`
344 console. error (
345 `creating experiment snapshot (one-time for arms=${ cfg . arms . map ( armId ). join ( ',' ) }; ` +
346 `trees cached: ${
347 cfg . arms
348 . filter (( a ) => a . treeCached )
349 . map (( a ) => a . name )
350 . join ( ',' ) || 'none'
351 })...`
352 )
353 try {
354 await sb ([
355 'create' ,
356 '--name' ,
357 vm,
358 '--runtime' ,
359 'node24' ,
360 '--vcpus' ,
361 '16' ,
362 '--timeout' ,
363 '1h' ,
364 '--non-persistent' ,
365 '--network-policy' ,
366 'allow-all' ,
367 '--tag' ,
368 'purpose=sandbox-bench' ,
369 '--silent' ,
370 ])
371 } catch (e) {
372 // Only a genuine capacity/plan rejection falls back to 8 vCPUs;
373 // anything else (auth, quota, network) must surface as itself.
374 if ( ! /vcpu/ i . test (e.message)) throw e
375 console. error ( '16 vCPUs unavailable, using 8' )
376 await sb ([
377 'create' ,
378 '--name' ,
379 vm,
380 '--runtime' ,
381 'node24' ,
382 '--vcpus' ,
383 '8' ,
384 '--timeout' ,
385 '1h' ,
386 '--non-persistent' ,
387 '--network-policy' ,
388 'allow-all' ,
389 '--tag' ,
390 'purpose=sandbox-bench' ,
391 '--silent' ,
392 ])
393 }
394 try {
395 for ( const tgz of new Set (
396 cfg.arms. filter (( a ) => ! a.treeCached). map (( a ) => a.nextTgz)
397 )) {
398 await sb ([ 'cp' , tgz, `${ vm }:/vercel/sandbox/${ path . basename ( tgz ) }` ])
399 }
400 for ( const arm of cfg.arms) {
401 if (arm.treeCached) {
402 console. error ( `uploading cached tree for ${ arm . name }...` )
403 await sbCpToVm (vm, arm.treeTgz, `/vercel/sandbox/tree-${ arm . name }.tgz` )
404 } else if (arm.tgz) {
405 await sb ([ 'cp' , arm.tgz, `${ vm }:/vercel/sandbox/arm-${ arm . name }.tgz` ])
406 }
407 }
408 // Cached trees: extract. Missing trees: install sequentially (the
409 // shared pnpm store dislikes concurrent cold installs), then build,
410 // sync, and warm concurrently — the builds dominate and parallelize
411 // across the 16 vCPUs. PHASE lines make the time budget visible.
412 const extractCached = cfg.arms
413 . filter (( a ) => a.treeCached)
414 . map (
415 ( a ) => `
416 echo "PHASE extract-${ a . name } $(date +%s)"
417 mkdir -p /vercel/sandbox/next-${ a . name } && tar -xzf /vercel/sandbox/tree-${ a . name }.tgz -C /vercel/sandbox/next-${ a . name }`
418 )
419 . join ( ' \n ' )
420 const installs = cfg.arms
421 . filter (( a ) => ! a.treeCached)
422 . map (
423 ( a ) => `
424 echo "PHASE install-${ a . name } $(date +%s)"
425 ${ a . tgz ? `mkdir -p /vercel/sandbox/arm-${ a . name } && tar -xzf /vercel/sandbox/arm-${ a . name }.tgz -C /vercel/sandbox/arm-${ a . name }` : ':'}
426 mkdir -p /vercel/sandbox/next-${ a . name } && cd /vercel/sandbox/next-${ a . name } && tar -xzf /vercel/sandbox/${ path . basename ( a . nextTgz ) } 2>/dev/null
427 pnpm install --frozen-lockfile >/tmp/i-${ a . name }.log 2>&1 || (tail -10 /tmp/i-${ a . name }.log; exit 1)`
428 )
429 . join ( ' \n ' )
430 const builds = cfg.arms
431 . filter (( a ) => ! a.treeCached)
432 . map (
433 ( a , i ) => `
434 (
435 set -e
436 cd /vercel/sandbox/next-${ a . name }
437 echo "PHASE build-${ a . name } $(date +%s)"
438 pnpm build >/tmp/b-${ a . name }.log 2>&1
439 ${ a . tgz ? `pnpm run sync-react --version "file:///vercel/sandbox/arm-${ a . name }/" >/tmp/s-${ a . name }.log 2>&1` : ': # vendored react, no sync'}
440 pnpm --filter=@next/font build >/dev/null 2>&1
441 echo "PHASE next-build-${ a . name } $(date +%s)"
442 pnpm --filter=next build >/tmp/n-${ a . name }.log 2>&1
443 ) &
444 BUILD_${ i }=$!`
445 )
446 . join ( ' \n ' )
447 const waits = cfg.arms
448 . filter (( a ) => ! a.treeCached)
449 . map (
450 ( a , i ) =>
451 `wait $BUILD_${ i } || (tail -10 /tmp/b-${ a . name }.log /tmp/s-${ a . name }.log /tmp/n-${ a . name }.log; exit 1)`
452 )
453 . join ( ' \n ' )
454 // Warm + verify runs per tree, sequential (they bind the same port).
455 const verifies = cfg.arms
456 . map (
457 ( a ) => `
458 cd /vercel/sandbox/next-${ a . name }
459 VER=$(grep -aom1 "[0-9.]*- \\ (canary \\ |experimental \\ )-[0-9a-f]*-[0-9]*" packages/next/dist/compiled/react-experimental/cjs/react.development.js || echo MISSING)
460 echo "tree ${ a . name } ver=$VER"
461 [ "$VER" != MISSING ]
462 echo "PHASE warm-${ a . name } $(date +%s)"
463 pnpm bench:render-pipeline --scenario=e2e --stream-mode=node --build=true --port=3720 --routes=${ cfg . routes } --warmup-requests=1 --serial-requests=2 --load-requests=2 --load-concurrency=1 --json-out=/tmp/warm.json --artifact-dir=/tmp/warm-art >/tmp/w.log 2>&1 || (tail -10 /tmp/w.log; exit 1)
464 rm -rf /tmp/warm-art /tmp/warm.json
465 ${ a . treeCached ? ':' : `echo "PHASE pack-${ a . name } $(date +%s)" && tar -czf /vercel/sandbox/tree-${ a . name }-out.tgz -C /vercel/sandbox/next-${ a . name } .`}
466 echo "tree ${ a . name } ready"`
467 )
468 . join ( ' \n ' )
469 await sbExec (
470 vm,
471 '55m' ,
472 `set -e \n npm i -g pnpm@10.33.0 >/dev/null 2>&1 \n ` +
473 `(while true; do echo "hb mem=$(free -m | awk '/^Mem/{print $3}')MB"; sleep 30; done) & HB=$! \n ` +
474 `${ extractCached } \n ${ installs } \n ${ builds } \n ${ waits } \n ${ verifies } \n kill $HB \n ` +
475 `echo "PHASE done $(date +%s)" \n ` +
476 `find /vercel/sandbox -maxdepth 1 -name '*.tgz' ! -name 'tree-*-out.tgz' -delete \n echo experiment ready` ,
477 'expsnap'
478 )
479 // Pull freshly built trees into the cache before snapshotting (the
480 // snapshot must not contain the multi-GB tarballs).
481 for ( const arm of cfg.arms) {
482 if ( ! arm.treeCached) {
483 console. error ( `caching built tree for ${ arm . name }...` )
484 // Temp + rename: concurrent launchers may cache the same tree.
485 const treeTmp = `${ arm . treeTgz }.tmp-${ process . pid }`
486 await sb ([
487 'cp' ,
488 `${ vm }:/vercel/sandbox/tree-${ arm . name }-out.tgz` ,
489 treeTmp,
490 ])
491 if (fs. existsSync (treeTmp) && fs. statSync (treeTmp).size >= 50_000_000 ) {
492 fs. renameSync (treeTmp, arm.treeTgz)
493 } else {
494 fs. rmSync (treeTmp, { force: true })
495 console. error (
496 `tree cache download for ${ arm . name } too small; skipping cache (snapshot unaffected)`
497 )
498 }
499 }
500 }
501 await sbExec (
502 vm,
503 '5m' ,
504 `rm -f /vercel/sandbox/tree-*.tgz /vercel/sandbox/tree-*-out.tgz; echo cleaned` ,
505 'expsnap'
506 )
507 return await takeSnapshot (vm, CACHE , key)
508 } finally {
509 await rmVm (vm)
510 }
511 }
512
513 // ---------------------------------------------------------------- stage
514
515 async function stage ( cfg , tmp ) {
516 const nextTgzBySha = new Map ()
517 for ( const arm of cfg.arms) {
518 if ( ! nextTgzBySha. has (arm.nextSha)) {
519 const tgz = path. join (tmp, `next-src-${ arm . nextSha . slice ( 0 , 12 ) }.tgz` )
520 await execFileP ( 'bash' , [
521 '-c' ,
522 `git -C ${ NEXT_REPO_LAZY () } archive ${ arm . nextSha } | gzip -1 > ${ tgz }` ,
523 ])
524 nextTgzBySha. set (arm.nextSha, tgz)
525 }
526 arm.nextTgz = nextTgzBySha. get (arm.nextSha)
527 if (arm.ref) {
528 arm. sha = (
529 await execFileP ( 'git' , [ '-C' , REACT_REPO_LAZY (), 'rev-parse' , arm.ref])
530 ).stdout. trim ()
531 await assertCiGreen (arm.sha, arm.name, cfg.allowUngated)
532 await ensureRefArm (arm)
533 }
534 }
535 }
536
537 // ------------------------------------------------------------------ run
538
539 async function runVm ( index , cfg , expSnap , outDir ) {
540 const vm = `sbench-${ cfg . label }-${ index }-${ Date . now (). toString ( 36 ) }`
541 const tag = `vm${ index }`
542 console. error ( `${ tag }: creating ${ vm } from experiment snapshot` )
543 writeStatus ({
544 vms: { ... status.state.vms, [vm]: { state: 'booting' , rows: 0 } },
545 })
546 await sb ([
547 'create' ,
548 '--name' ,
549 vm,
550 '--snapshot' ,
551 expSnap,
552 '--vcpus' ,
553 '8' ,
554 '--timeout' ,
555 '5h' ,
556 '--non-persistent' ,
557 '--network-policy' ,
558 'allow-all' ,
559 '--tag' ,
560 'purpose=sandbox-bench' ,
561 '--silent' ,
562 ])
563 try {
564 const [ base , cand ] = cfg.arms. map (( a ) => a.name)
565 const total = cfg.blocks * cfg.runs
566 const benchArgs = ( extra ) =>
567 `--scenario=e2e --stream-mode=node --build=false --port=$PORT ` +
568 `--routes=${ cfg . routes } --warmup-requests=${ cfg . warmup } --serial-requests=${ cfg . serial } ` +
569 `--load-requests=${ cfg . loadRequests } --load-concurrency=${ cfg . loadConcurrency } ` +
570 `${ cfg . isolateRoutes ? '--isolate-routes=true ' : ''}${ extra }`
571 // Both trees are pre-built in the snapshot; a run is pure
572 // measurement. ABBA: alternate which arm goes first per run so
573 // linear drift cancels within pairs, not just across them.
574 const loop = `set -e
575 VMINDEX=${ index }
576 CPU=$(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2- | sed 's/^ //')
577 : > /vercel/sandbox/results.jsonl
578 for arm in ${ base } ${ cand }; do
579 V=$(grep -aom1 "[0-9.]*- \\ (canary \\ |experimental \\ )-[0-9a-f]*-[0-9]*" /vercel/sandbox/next-$arm/packages/next/dist/compiled/react-experimental/cjs/react.development.js || echo MISSING)
580 for f in ${ FP_FILES . map (( f ) => `/vercel/sandbox/next-$arm/${ f }` ). join ( ' ' ) }; do [ -s "$f" ] || { echo "FP file missing: $f"; exit 1; }; done
581 F=$(cat ${ FP_FILES . map (( f ) => `/vercel/sandbox/next-$arm/${ f }` ). join ( ' ' ) } | sha256sum | cut -c1-12)
582 echo "tree $arm ver=$V fp=$F"; [ "$V" != MISSING ]
583 eval "VER_$arm=$V; FP_$arm=$F"
584 done
585 # Identical fingerprints can be legitimate (arms differing only in
586 # files outside FP_FILES, e.g. client-only changes), so warn, not fail.
587 if [ "$FP_${ base }" = "$FP_${ cand }" ]; then
588 echo "WARNING: arms fingerprint identically ($FP_${ base }) — the hashed server bundles are byte-identical; verify the arms differ where intended"
589 fi
590 for run in $(seq 1 ${ total }); do
591 # Alternate within the boot AND stagger by VM index: with an odd run
592 # count, otherwise every boot gives the same arm the cold first slot
593 # and boot-level inference reads that shared bias as signal.
594 if [ $(((run + VMINDEX) % 2)) = 1 ]; then ORDER="${ cand } ${ base }"; else ORDER="${ base } ${ cand }"; fi
595 for arm in $ORDER; do
596 # One port per arm: a not-quite-dead server from a previous run can
597 # then never be measured as the other arm.
598 if [ "$arm" = "${ base }" ]; then PORT=3720; else PORT=3721; fi
599 cd /vercel/sandbox/next-$arm
600 ${ cfg . benchEnv ? `export ${ cfg . benchEnv }` : ':'}
601 pnpm bench:render-pipeline ${ benchArgs ( '' ) } \
602 --json-out=/tmp/r.json --artifact-dir=/tmp/art-$arm-r$run >/tmp/bench.log 2>&1 \
603 || (tail -20 /tmp/bench.log; exit 1)
604 rm -rf /tmp/art-$arm-r$run
605 eval "FP= \\ $FP_$arm; VER= \\ $VER_$arm"
606 node -e '
607 const [,run,arm,fp,ver,cpu]=process.argv;
608 const j=require("/tmp/r.json");
609 const docs=new Map((j.fullResults[0].routeDocuments??[]).map(d=>[d.route,d]));
610 for (const rr of j.fullResults[0].routeResults) {
611 if (!rr.latency) continue;
612 const row={block:+run,arm,run:1,fp,ver,cpu,route:rr.route,
613 phase:rr.phase,rps:rr.throughputRps,median:rr.latency.median,
614 mean:rr.latency.mean,p95:rr.latency.p95};
615 // Optional metrics are OMITTED when absent — a zero would pair
616 // against a real value as a fabricated -100% claim.
617 if (rr.latency.p99>0) row.p99=rr.latency.p99;
618 if (rr.ttfb&&rr.ttfb.median>0) row.ttfb=rr.ttfb.median;
619 if (rr.serverRssMb>0) row.rss=rr.serverRssMb;
620 if (rr.serverRssHwMb>0) row.rssHw=rr.serverRssHwMb;
621 const d=docs.get(rr.route)??{};
622 if (d.bytes>0) row.docKb=d.bytes/1024;
623 if (d.gzipBytes>0) row.gzipKb=d.gzipBytes/1024;
624 if (d.inlineFlightBytes>0) row.flightKb=d.inlineFlightBytes/1024;
625 // Failed requests inflate rps and vanish from latency: surface.
626 if (rr.errors>0) row.errors=rr.errors;
627 console.log(JSON.stringify(row));
628 }
629 ' "$run" "$arm" "$FP" "$VER" "$CPU" > /tmp/rows.txt
630 cat /tmp/rows.txt >> /vercel/sandbox/results.jsonl
631 sed 's/^/ROW /' /tmp/rows.txt
632 echo "run $run $arm done"
633 done
634 done
635 wc -l /vercel/sandbox/results.jsonl`
636 let vmRows = 0
637 const out = await runDetached (
638 vm,
639 tag,
640 loop,
641 ( row ) => {
642 vmRows ++
643 interimRows. push ({ vm: index, ... row })
644 writeStatus ({
645 vms: {
646 ... status.state.vms,
647 [vm]: { state: 'measuring' , rows: vmRows },
648 },
649 })
650 cfg. live (index, row)
651 },
652 110
653 )
654 writeStatus ({
655 vms: {
656 ... status.state.vms,
657 [vm]: { ... status.state.vms[vm], state: 'collecting' },
658 },
659 })
660 const local = path. join (outDir, `results-vm${ index }.jsonl` )
661 await sb ([ 'cp' , `${ vm }:/vercel/sandbox/results.jsonl` , local])
662 const remoteCount = Number (
663 out. match ( /( \d + ) \/ vercel \/ sandbox \/ results \. jsonl/ )?.[ 1 ] ?? NaN
664 )
665 const localCount = fs
666 . readFileSync (local, 'utf8' )
667 . trim ()
668 . split ( ' \n ' )
669 . filter (Boolean). length
670 if ( ! Number. isFinite (remoteCount) || localCount !== remoteCount) {
671 throw new Error (
672 `${ tag }: downloaded ${ localCount } rows, remote reported ${ remoteCount } — truncated transfer`
673 )
674 }
675
676 if (cfg.profile) {
677 writeStatus ({
678 vms: {
679 ... status.state.vms,
680 [vm]: { ... status.state.vms[vm], state: 'profiling' },
681 },
682 })
683 // Profiled passes run strictly AFTER the timed runs — profiling
684 // overhead must never touch the numbers.
685 // Within one VM the second arm runs warmer (page cache, CPU
686 // governor) — a uniform few-percent deflation of every function
687 // in its profile. Alternating order across VMs cancels the drift
688 // in cross-VM aggregates.
689 const profOrder = index % 2 === 1 ? `${ cand } ${ base }` : `${ base } ${ cand }`
690 const prof = `set -e
691 for arm in ${ profOrder }; do
692 if [ "$arm" = "${ base }" ]; then PORT=3720; else PORT=3721; fi
693 cd /vercel/sandbox/next-$arm
694 pnpm bench:render-pipeline ${ benchArgs ( '--capture-cpu' ) } \
695 --json-out=/tmp/pr.json --artifact-dir=/vercel/sandbox/prof-$arm >/tmp/prof.log 2>&1 \
696 || (tail -10 /tmp/prof.log; exit 1)
697 echo "profiled $arm"
698 done
699 # Lets profile analysis split by capture order without re-deriving it
700 # from VM index parity.
701 echo "${ profOrder }" > /vercel/sandbox/prof-order.txt
702 cd /vercel/sandbox && tar -czf profiles.tgz prof-*`
703 // Profile capture is best-effort: a failed pass or transfer on one VM
704 // must not kill collection for the whole run (the timed results are
705 // already on disk at this point). Each VM extracts into its own
706 // subdirectory so VMs don't overwrite each other's prof-<arm> dirs.
707 try {
708 await sbExec (vm, '30m' , prof, `${ tag }:prof` )
709 const profTgz = path. join (outDir, `profiles-vm${ index }.tgz` )
710 await sb ([ 'cp' , `${ vm }:/vercel/sandbox/profiles.tgz` , profTgz])
711 if ( ! fs. existsSync (profTgz) || fs. statSync (profTgz).size === 0 ) {
712 throw new Error ( 'profile tarball missing or empty after cp' )
713 }
714 const vmProfDir = path. join (outDir, `prof-vm${ index }` )
715 fs. mkdirSync (vmProfDir, { recursive: true })
716 await execFileP ( 'tar' , [ '-xzf' , profTgz, '-C' , vmProfDir])
717 console. error ( `${ tag }: profiles in ${ vmProfDir }` )
718 } catch (profErr) {
719 console. error (
720 `${ tag }: profile capture failed (timed results unaffected): ${ profErr . message }`
721 )
722 }
723 }
724 writeStatus ({
725 vms: {
726 ... status.state.vms,
727 [vm]: { ... status.state.vms[vm], state: 'done' },
728 },
729 })
730 return local
731 } finally {
732 if ( ! cfg.keep) await rmVm (vm)
733 else console. error ( `${ tag }: kept ${ vm }` )
734 }
735 }
736
737 // -------------------------------------------------------------- analyze
738
739 function analyze ( cfg ) {
740 // Collected JSONL + profiles land in the results db first; the stats
741 // read the db and nothing else. Import is idempotent, verify is
742 // mechanical (sqlite integrity, pairing shape, artifact hashes).
743 const db = openDb ()
744 const { runId , samples , artifacts } = importRun (db, outDir)
745 const { failures , notes } = verifyDb (db, runId)
746 for ( const n of notes) console. log ( `note: ${ n }` )
747 if (failures. length ) {
748 throw new Error ( `results db verify FAILED: \n ${ failures . join ( ' \n ' ) }` )
749 }
750 console. error (
751 `results db: ${ samples } samples, ${ artifacts } artifacts as ${ runId } (verify ok)`
752 )
753 const rows = loadRows (db, runId)
754 const [ base , cand ] = cfg.arms. map (( a ) => a.name)
755 printRunContext (( m ) => console. log (m), cfg.runContext)
756 // ttfb/rss/rssHw only exist on next refs carrying the bench-client-
757 // metrics harness; metrics without data are skipped.
758 analyzeE2eRows (rows, base, cand, [
759 'rps' ,
760 'median' ,
761 'mean' ,
762 'p95' ,
763 'ttfb' ,
764 'docKb' ,
765 'gzipKb' ,
766 'flightKb' ,
767 'rss' ,
768 'rssHw' ,
769 ])
770 }
771
772 // ----------------------------------------------------------------- main
773
774 // Dry-run: resolve as much as possible with local-only operations and
775 // print the execution plan instead of touching the sandbox. Degrades
776 // gracefully when prerequisites (clones, config) are missing so it can
777 // be used to sanity-check a setup before committing to a real run.
778 async function plan ( cfg ) {
779 const lines = []
780 lines. push (
781 `mode: ${ cfg . pr ? `react PR ${ cfg . pr }` : cfg . arms . length ? 'react A/B' : cfg . nextPr ? `next PR ${ cfg . nextPr }` : 'next A/B'}`
782 )
783 lines. push (
784 `scope: team=${ CONFIG . team ?? '<UNSET — ask user, then: node config.mjs set team=... project=...>'} project=${ CONFIG . project ?? '<UNSET>'}`
785 )
786 lines. push ( `next repo: ${ NEXT_REPO_LAZY () }` )
787 if (cfg.pr || cfg.arms. length || cfg.reactRef) {
788 const rr = REACT_REPO_LAZY ()
789 lines. push (
790 `react repo: ${ rr }${ fs . existsSync ( path . join ( rr , '.git' )) ? '' : ' (would clone on real run)'}`
791 )
792 } else {
793 lines. push ( 'react: vendored in each Next ref (no react checkout needed)' )
794 }
795 let arms
796 try {
797 arms = await resolveArms (cfg)
798 for ( const arm of arms) {
799 if ( ! arm.ref) {
800 lines. push (
801 `arm ${ arm . name }: react=vendored next=${ arm . nextSha . slice ( 0 , 12 ) }`
802 )
803 continue
804 }
805 let reactSha = '?'
806 try {
807 reactSha = (
808 await execFileP ( 'git' , [
809 '-C' ,
810 REACT_REPO_LAZY (),
811 'rev-parse' ,
812 arm.ref,
813 ])
814 ).stdout
815 . trim ()
816 . slice ( 0 , 12 )
817 } catch {}
818 const cached =
819 reactSha !== '?' &&
820 fs. existsSync (path. join ( CACHE , `arm-${ reactSha }.tgz` ))
821 lines. push (
822 `arm ${ arm . name }: react=${ arm . ref } (${ reactSha }${ cached ? ', build cached' : ', would build remotely ~15m'}) next=${ arm . nextSha . slice ( 0 , 12 ) }`
823 )
824 lines. push (
825 ` CI gate: react commit must be CI-green (or --allow-ungated after sandbox-gate.mjs)`
826 )
827 }
828 } catch (e) {
829 lines. push (
830 `arms: unresolved in dry-run (${ e . message . split ( ' \n ' )[ 0 ]. slice ( 0 , 120 ) })`
831 )
832 }
833 lines. push (
834 `then: experiment snapshot (cached by content key; ~45m if cold) -> ` +
835 `${ cfg . vms } VMs x ${ cfg . blocks * cfg . runs } paired ABBA runs, routes ${ cfg . routes }` +
836 `${ cfg . isolateRoutes ? ' (isolated)' : ''}${ cfg . benchEnv ? ` env ${ cfg . benchEnv }` : ''}`
837 )
838 lines. push (
839 `then: boot-level analysis (n=${ cfg . vms } boots) -> claims at p<0.01 on A/A-validated infra`
840 )
841 console. log (lines. map (( l ) => `[dry-run] ${ l }` ). join ( ' \n ' ))
842 }
843
844 const cfg = parseArgs ()
845 if (cfg.dryRun) {
846 await plan (cfg)
847 process. exit ( 0 )
848 }
849 fs. mkdirSync ( CACHE , { recursive: true })
850 const tmp = fs. mkdtempSync (path. join (os. tmpdir (), 'sandbox-e2e-' ))
851 const outDir = path. join ( CACHE , `run-${ cfg . label }-${ Date . now (). toString ( 36 ) }` )
852 fs. mkdirSync (outDir, { recursive: true })
853 // stdout, not stderr: task UIs preview stdout, and these are the lines
854 // a human watching the task needs.
855 console. log ( `run dir: ${ outDir }` )
856 status.file = path. join (outDir, 'status.json' )
857 // Long phases (remote builds, snapshot assembly) are otherwise silent
858 // on stdout, which reads as a hung task in any UI that previews output.
859 // A periodic one-line digest keeps the task legible without spam.
860 const interimRows = []
861 // Interim display: running effect + directional confidence
862 // P(effect > 0 | boots so far) — the Student-t posterior under a flat
863 // prior, i.e. "how sure are we the candidate is actually faster".
864 // Display only; runs complete their allocation and claims come from
865 // the final analysis.
866 function interimSummary () {
867 if (interimRows. length === 0 || ! status.state.arms) return ''
868 const [ base , cand ] = cfg.arms. map (( a ) => a.name)
869 const cells = []
870 for ( const route of cfg.routes. split ( ',' )) {
871 for ( const phase of [ 'single-client' , 'under-load' ]) {
872 const perBoot = []
873 for ( const vmIdx of new Set (interimRows. map (( r ) => r.vm))) {
874 const deltas = []
875 for ( const r of interimRows. filter (
876 ( x ) =>
877 x.vm === vmIdx &&
878 x.route === route &&
879 x.phase === phase &&
880 x.arm === cand
881 )) {
882 const b = interimRows. find (
883 ( x ) =>
884 x.vm === vmIdx &&
885 x.route === route &&
886 x.phase === phase &&
887 x.arm === base &&
888 x.block === r.block &&
889 x.run === r.run
890 )
891 if (b && b.rps > 0 && r.rps > 0 ) deltas. push ((r.rps - b.rps) / b.rps)
892 }
893 if (deltas. length )
894 perBoot. push (deltas. reduce (( a , b ) => a + b, 0 ) / deltas. length )
895 }
896 if (perBoot. length < 4 ) continue
897 const mean = perBoot. reduce (( a , b ) => a + b, 0 ) / perBoot. length
898 const pTwo = tTestP (perBoot)
899 const conf = Math. max ( 1 - pTwo / 2 , pTwo / 2 )
900 const confStr = conf >= 0.999 ? '>99.9%' : `${ ( conf * 100 ). toFixed ( 0 ) }%`
901 const label = `${ route } ${ phase === 'single-client' ? 'serial' : 'load'}:`
902 cells. push (
903 ` ${ label . padEnd ( 19 ) } ${ mean > 0 ? '+' : ''}${ ( mean * 100 ). toFixed ( 1 ) }% rps (${ confStr } confidence)`
904 )
905 }
906 }
907 if (cells. length === 0 ) return ''
908 const boots = new Set (interimRows. map (( r ) => r.vm)).size
909 return ` \n interim vs base (${ boots } boots): \n ` + cells. join ( ' \n ' )
910 }
911 const digest = setInterval (() => {
912 const vms = Object. values (status.state.vms ?? {})
913 const rows = vms. reduce (( a , v ) => a + (v.rows ?? 0 ), 0 )
914 const states = {}
915 for ( const v of vms) states[v.state] = (states[v.state] ?? 0 ) + 1
916 const vmSummary = Object. entries (states)
917 . map (([ k , n ]) => `${ n } ${ k }` )
918 . join ( ', ' )
919 console. log (
920 `progress: ${ status . state . phase }` +
921 (status.state.rowsExpected
922 ? ` — rows ${ rows }/${ status . state . rowsExpected }`
923 : '' ) +
924 (vmSummary ? ` (${ vmSummary })` : '' ) +
925 interimSummary ()
926 )
927 }, 120_000 )
928 digest. unref ?.()
929 writeStatus ({
930 label: cfg.label,
931 phase: 'resolving arms' ,
932 pid: process.pid,
933 startedAt: new Date (). toISOString (),
934 vms: {},
935 rowsExpected: null ,
936 })
937 try {
938 cfg.arms = await resolveArms (cfg)
939 cfg.runContext = await describeRun (cfg)
940 printRunContext (( m ) => console. error (m), cfg.runContext)
941 fs. writeFileSync (
942 path. join (outDir, 'meta.json' ),
943 JSON . stringify (
944 {
945 base: cfg.arms[ 0 ].name,
946 cand: cfg.arms[ 1 ].name,
947 label: cfg.label,
948 nextRef: cfg.nextRef,
949 vms: cfg.vms,
950 blocks: cfg.blocks,
951 runs: cfg.runs,
952 routes: cfg.routes,
953 pr: cfg.runContext.pr,
954 arms: cfg.runContext.arms,
955 benchEnv: cfg.benchEnv || undefined ,
956 },
957 null ,
958 2
959 )
960 )
961 cfg.live = makeLive (
962 cfg.arms[ 0 ].name,
963 [ 'rps' , 'median' , 'p95' ],
964 ( r ) => `${ r . route } ${ r . phase }`
965 )
966 await stage (cfg, tmp)
967 console. error (
968 `arms: ${ cfg . arms . map (( a ) => `${ a . name }=${ armId ( a ) }` ). join ( ' ' ) }`
969 )
970 writeStatus ({
971 phase: 'building arms + experiment snapshot' ,
972 arms: cfg.arms. map (( a ) => `${ a . name }=${ armId ( a ) }` ),
973 })
974 const expSnap = await ensureExperimentSnapshot (cfg)
975 if (cfg.prepare) {
976 // A prepare run launches no measurement VMs; anything but a
977 // terminal phase here makes bench-status prescribe collecting
978 // data that never existed.
979 writeStatus ({ phase: 'prepared (caches only, no measurement)' , expSnap })
980 console. error (
981 `prepared: arms + experiment snapshot ${ expSnap }; exiting (--prepare)`
982 )
983 process. exit ( 0 )
984 }
985 writeStatus ({
986 phase: 'measuring' ,
987 expSnap,
988 rowsExpected:
989 cfg.vms * cfg.blocks * cfg.runs * 2 * cfg.routes. split ( ',' ). length * 2 ,
990 })
991 await Promise . all (
992 Array. from ({ length: cfg.vms }, ( _ , i ) => runVm (i, cfg, expSnap, outDir))
993 )
994 writeStatus ({ phase: 'analyzing' })
995 console. error ( `results in ${ outDir }` )
996 analyze (cfg)
997 writeStatus ({ phase: 'done' })
998 } catch (err) {
999 // Leave a machine-readable trace: bench-status.mjs reports dead runs
1000 // and the right recovery action from this.
1001 writeStatus ({
1002 phase: 'failed' ,
1003 error: String ((err && err.message) || err). slice ( 0 , 500 ),
1004 })
1005 throw err
1006 } finally {
1007 fs. rmSync (tmp, { recursive: true , force: true })
1008 }