Setting the file. One moment.
Bench Stats · Sandbox Bench · vercel/next.js · Skills Docs
Back to the top of the page 24.4
Bench Common
scripts/ bench-stats.mjs
JavaScript · 230 lines · 9 KB
4.303
,
3.182
,
2.776
,
2.571
,
2.447
,
2.365
,
2.306
,
2.262
,
2.228
,
2.201
,
14 2.179 , 2.16 , 2.145 , 2.131 , 2.12 , 2.11 , 2.101 , 2.093 , 2.086 , 2.08 , 2.074 ,
15 2.069 , 2.064 , 2.06 , 2.056 , 2.052 , 2.048 , 2.045 , 2.042 ,
16 ]
17
18 function tCritical975 ( df ) {
19 if (df < 1 ) return Infinity
20 if (df <= 30 ) return T975 [df - 1 ]
21 return 1.96 + 2.4 / df // adequate approximation past df=30
22 }
23
24 // Two-sided p for a one-sample t test against zero, via numerical
25 // integration of the t pdf (small-df accuracy is what matters here —
26 // boots are few).
27 export function tTestP ( values ) {
28 const n = values. length
29 if (n < 2 ) return 1
30 if (values. some (( v ) => ! Number. isFinite (v))) return 1
31 const mean = values. reduce (( a , b ) => a + b, 0 ) / n
32 const sd = Math. sqrt (
33 values. reduce (( a , b ) => a + (b - mean) ** 2 , 0 ) / (n - 1 )
34 )
35 if (sd === 0 ) return mean === 0 ? 1 : 0
36 const t = Math. abs (mean / (sd / Math. sqrt (n)))
37 const df = n - 1
38 if (df === 1 ) {
39 // Student t with df=1 is Cauchy; the closed form avoids the fat
40 // tail truncating a numerical integration.
41 return Math. min ( 1 , Math. max ( 0 , 1 - ( 2 / Math. PI ) * Math. atan (t)))
42 }
43 if ( ! Number. isFinite (t) || t > 45 ) {
44 // p underflows well past any claim threshold; also guards the
45 // integration loop below, whose step size vanishes against huge t.
46 return 0
47 }
48 const pdf = ( x ) => Math. exp ( - ((df + 1 ) / 2 ) * Math. log ( 1 + (x * x) / df))
49 let integral = 0
50 const STEP = 0.001
51 for ( let x = t; x < t + 60 ; x += STEP ) integral += pdf (x + STEP / 2 ) * STEP
52 let norm = 0
53 for ( let x = 0 ; x < 80 ; x += STEP ) norm += pdf (x + STEP / 2 ) * STEP
54 return Math. min ( 1 , integral / norm)
55 }
56
57 // Anytime-valid confidence sequence for a running mean (asymptotic CS,
58 // Waudby-Smith & Ramdas). Unlike a t-CI, this interval is valid at
59 // EVERY peek simultaneously, so interim displays built on it cannot
60 // manufacture significance through repeated looking. Tuned to be
61 // tightest around ~12 boots.
62 export function confidenceSeq ( values , alpha = 0.05 ) {
63 const n = values. length
64 // Below 6 samples the estimated variance is too unstable for the
65 // asymptotic guarantee; show nothing rather than something wrong.
66 if (n < 6 ) return null
67 const mean = values. reduce (( a , b ) => a + b, 0 ) / n
68 let sd = Math. sqrt (values. reduce (( a , b ) => a + (b - mean) ** 2 , 0 ) / (n - 1 ))
69 if ( ! Number. isFinite (sd)) return null
70 // Small-sample variance inflation (t-style): keeps the sequence
71 // honest at the n this harness actually runs (6..32 boots).
72 sd *= Math. sqrt ((n - 1 ) / Math. max ( 1 , n - 3 ))
73 const nOpt = 16
74 const rho2 = ( 2 * Math. log ( 2 / alpha)) / nOpt
75 const width =
76 sd *
77 Math. sqrt (
78 (( 2 * (n * rho2 + 1 )) / (n * n * rho2)) *
79 Math. log ((Math. sqrt (n * rho2 + 1 ) * 2 ) / alpha)
80 )
81 return { mean, lo: mean - width, hi: mean + width, n }
82 }
83
84 // What each metric measures and which direction is an improvement.
85 // Deltas are always relative (%), so the unit answers "% of what".
86 export const METRICS = {
87 rps: { unit: 'req/s' , better: 'higher' },
88 docKb: { unit: 'KB' , better: 'lower' },
89 gzipKb: { unit: 'KB' , better: 'lower' },
90 flightKb: { unit: 'KB' , better: 'lower' },
91 median: { unit: 'ms' , better: 'lower' },
92 mean: { unit: 'ms' , better: 'lower' },
93 p95: { unit: 'ms' , better: 'lower' },
94 p99: { unit: 'ms' , better: 'lower' },
95 ttfb: { unit: 'ms' , better: 'lower' },
96 gcMs: { unit: 'ms' , better: 'lower' },
97 p50: { unit: 'ms' , better: 'lower' },
98 rss: { unit: 'MB' , better: 'lower' },
99 heapMb: { unit: 'MB' , better: 'lower' },
100 rssHw: { unit: 'MB peak' , better: 'lower' },
101 }
102
103 // One metric in one phase: per-boot arrays of paired deltas in, verdict out.
104 export function bootLevelStats ( perBootDeltas ) {
105 const boots = perBootDeltas. filter (( d ) => d. length > 0 )
106 const bootMeans = boots. map (( d ) => d. reduce (( a , b ) => a + b, 0 ) / d. length )
107 const n = bootMeans. length
108 if (n === 0 ) return null
109 const mean = bootMeans. reduce (( a , b ) => a + b, 0 ) / n
110 const all = boots. flat ()
111 const result = {
112 mean,
113 boots: n,
114 bootMeans,
115 pairs: all. length ,
116 withinP: tTestP (all),
117 }
118 if (n >= 2 ) {
119 const sd = Math. sqrt (
120 bootMeans. reduce (( a , b ) => a + (b - mean) ** 2 , 0 ) / (n - 1 )
121 )
122 result.ci95 = ( tCritical975 (n - 1 ) * sd) / Math. sqrt (n)
123 result.p = tTestP (bootMeans)
124 } else {
125 result.ci95 = Infinity
126 result.p = 1
127 }
128 return result
129 }
130
131 export function formatStat ( name , candName , baseName , s ) {
132 if (s === null ) return ` ${ name . padEnd ( 6 ) } (no data)`
133 const pct = ( x ) => `${ ( x * 100 ). toFixed ( 1 ) }%`
134 const ci = s.ci95 === Infinity ? '±∞' : `±${ ( s . ci95 * 100 ). toFixed ( 1 ) }`
135 const m = METRICS [name]
136 const meta = m
137 ? ` [${ m . unit }; ${ s . mean > 0 === ( m . better === 'higher' ) ? 'IMPROVEMENT' : 'regression'} if real]`
138 : ''
139 // Byte metrics are deterministic per build, so a ±0.0 near-zero cell
140 // with p=0 is normal; the absolute values say whether it matters.
141 const abs =
142 s.absBase !== undefined && m?.unit === 'KB'
143 ? ` [${ s . absBase . toFixed ( 1 ) }KB -> ${ s . absCand . toFixed ( 1 ) }KB]`
144 : ''
145 return (
146 ` ${ name . padEnd ( 6 ) } ${ candName } vs ${ baseName }: ${ pct ( s . mean ) } ${ ci }${ meta }${ abs } ` +
147 `(boots=${ s . boots }${ s . boots < 3 ? ' — TOO FEW FOR CLAIMS' : ''}, p=${ s . p . toFixed ( 4 ) })` +
148 ` perBoot ${ s . bootMeans . map (( m ) => pct ( m )). join ( ' ' ) }` +
149 ` [pairs=${ s . pairs } within-run p=${ s . withinP . toFixed ( 4 ) } — diagnostic only]`
150 )
151 }
152
153 // E2e rows: {vm, arm, block, run, route, phase, <metrics...>}. Pairs are
154 // (vm, block, run); the boot is the vm. Returns nothing; prints.
155 export function analyzeE2eRows ( rows , baseName , candName , metrics ) {
156 const fps = {}
157 for ( const r of rows)
158 (fps[r.arm] ||= new Set ()). add (r.ver ? `${ r . fp }/${ r . ver }` : `${ r . fp }` )
159 console. log (
160 ` \n fingerprints: ${ baseName }=${ [ ... ( fps [ baseName ] ?? [])]. join ( ',' ) } ` +
161 `${ candName }=${ [ ... ( fps [ candName ] ?? [])]. join ( ',' ) }`
162 )
163 if ((fps[baseName]?.size ?? 0 ) !== 1 || (fps[candName]?.size ?? 0 ) !== 1 ) {
164 // A VM measured the wrong build; numbers would look official and be
165 // meaningless. Refuse instead of printing stats with a warning.
166 console. log (
167 '!! inconsistent fingerprints within an arm — RESULTS INVALID, no stats'
168 )
169 process.exitCode = 1
170 return false
171 }
172 const baseFp = [ ... fps[baseName]][ 0 ]?. split ( '/' )[ 0 ]
173 const candFp = [ ... fps[candName]][ 0 ]?. split ( '/' )[ 0 ]
174 if (baseFp !== undefined && candFp !== undefined ) {
175 console. log (
176 baseFp === candFp
177 ? 'fingerprint files byte-identical between arms (A/A for those files; ' +
178 'arms may still differ elsewhere — check version strings)'
179 : 'arms differ (A/B mode)'
180 )
181 }
182
183 // "Absent" must be distinguishable from "identical": metrics the run
184 // never captured are named at the end instead of silently missing.
185 const captured = new Set ()
186 for ( const phase of [
187 ...new Set (rows. map (( r ) => `${ r . route ?? ''} ${ r . phase }` )),
188 ]. sort ()) {
189 console. log ( ` \n ${ phase }` )
190 const inPhase = ( r ) => `${ r . route ?? ''} ${ r . phase }` === phase
191 for ( const metric of metrics) {
192 const perBoot = []
193 const baseVals = []
194 const candVals = []
195 for ( const vm of [ ...new Set (rows. map (( r ) => r.vm))]) {
196 const deltas = []
197 for ( const r of rows. filter (
198 ( x ) => x.vm === vm && inPhase (x) && x.arm === candName
199 )) {
200 const b = rows. find (
201 ( x ) =>
202 x.vm === vm &&
203 inPhase (x) &&
204 x.arm === baseName &&
205 x.block === r.block &&
206 x.run === r.run
207 )
208 if (b && b[metric] > 0 && r[metric] > 0 ) {
209 deltas. push ((r[metric] - b[metric]) / b[metric])
210 baseVals. push (b[metric])
211 candVals. push (r[metric])
212 }
213 }
214 perBoot. push (deltas)
215 }
216 const s = bootLevelStats (perBoot)
217 if (s !== null && s.pairs > 0 ) {
218 captured. add (metric)
219 s.absBase = baseVals. reduce (( a , b ) => a + b, 0 ) / baseVals. length
220 s.absCand = candVals. reduce (( a , b ) => a + b, 0 ) / candVals. length
221 console. log ( formatStat (metric, candName, baseName, s))
222 }
223 }
224 }
225 const absent = metrics. filter (( m ) => ! captured. has (m))
226 if (absent. length > 0 ) {
227 console. log ( ` \n not captured on this run: ${ absent . join ( ', ' ) }` )
228 }
229 return true
230 }