Setting the file. One moment.
Srm Check · Debugging Experiments · PostHog/skills · Skills Docs
Back to the top of the page def wilson_interval
— line 186
This file
Number 40.4
Position 4 of 4
Type Python
Size 30 KB
Lines 688 scripts/ srm_check.py
Python · 688 lines · 30 KB
16 measures how much something moved users between arms after assignment. The script reports both
17 components, each with a significance test, and only names a side when one of them both dominates
18 the gap and is statistically distinguishable from zero. Otherwise it says so.
19
20 Algorithm is byte-exact with the PostHog implementation in
21 `rust/feature-flags/src/flags/flag_matching.rs` (get_matching_variant) and
22 `flag_matching_utils.rs` (calculate_hash). Run `--selftest` first — it checks every part a wrong
23 verdict would come from, and exits non-zero on any mismatch:
24
25 hash pipeline replayed against the repo's golden vectors
26 variant hash key the `{flag_key}.` prefix and the `variant` salt
27 variant walk stored order and the strict `<` bound
28 statistics chi-squared tail against known critical values, Wilson interval
29 verdict synthetic pure-capture and pure-assignment samples route correctly
30
31 Stdlib only (hashlib, csv, math, argparse) — no PostHog install required. distinct_ids are
32 often emails; run this customer-side and paste back only the aggregate lines it prints.
33
34 ./srm_check.py --selftest
35 ./srm_check.py --flag-key my-flag --variants-file variants.json --csv exposures.csv
36
37 Prefer --variants-file: save the flag's `filters.multivariate.variants` array to a file and pass
38 the path. Variant keys are only charset-validated in the PostHog UI, not by the API, so a key
39 reaching you through a ticket can contain shell metacharacters or quotes — keep it out of the
40 command line entirely rather than trying to quote it. --variants is the convenience form for keys
41 you have already eyeballed.
42
43 The CSV is the export query from the decisive test: a header row plus
44 `distinct_id,recorded_variant,variants_seen` (override names with --id-col / --variant-col /
45 --variants-seen-col; `variants_seen` may be absent). The id column must hold the identifier
46 production hashed: the group key for a group-aggregated flag, or `$device_id` (coalesced to
47 distinct_id when empty) for a device-ID-bucketed flag (`bucketing_identifier == "device_id"`) —
48 otherwise the distinct_id. Feeding the wrong identifier fabricates disagreements; the selftested
49 chance-agreement guard below catches the worst case, but not a subtle one.
50 """
51 from __future__ import annotations
52
53 import argparse
54 import csv
55 import hashlib
56 import json
57 import math
58 import sys
59 from dataclasses import dataclass
60
61 # 0xfffffffffffffff == 15 hex digits == LONG_SCALE in flag_matching_utils.rs
62 __LONG_SCALE__ = 0x FFFFFFFFFFFFFFF
63
64 # PostHog treats an SRM as real below this p (see the chi-squared section in pulling-the-data.md).
65 SRM_ALPHA = 0.001
66 # A component has to carry at least this much of the gap before it names a side on its own.
67 DOMINANT_SHARE = 2.0 / 3.0
68
69
70 def hash_of (hash_key: str ) -> float :
71 """The pipeline half of calculate_hash() in flag_matching_utils.rs: first 15 hex
72 chars of sha1(hash_key), divided by LONG_SCALE. Deterministic, in [0, 1).
73
74 SHA1 here is a compatibility requirement, not a security choice: it is the hash
75 PostHog's flag matcher buckets users with, so this has to reproduce it bit for bit.
76 Do not take semgrep's SHA256 autofix — it would still compute a number and still
77 print a verdict, just a wrong one, which is the worst failure this script has."""
78 # nosemgrep: python.lang.security.insecure-hash-algorithms.insecure-hash-algorithm-sha1 (reproduces flag bucketing, not a signature)
79 return int (hashlib.sha1(hash_key.encode( "utf-8" )).hexdigest()[: 15 ], 16 ) / __LONG_SCALE__
80
81
82 def calculate_hash (prefix: str , identifier: str , salt: str = "" ) -> float :
83 """Mirrors calculate_hash() in flag_matching_utils.rs, which concatenates
84 prefix + identifier + salt before hashing."""
85 return hash_of( f " { prefix }{ identifier }{ salt } " )
86
87
88 def variant_hash_key (flag_key: str , identifier: str ) -> str :
89 """The exact string get_hash() feeds to sha1 for the variant walk: the `{flag_key}.`
90 prefix, the identifier, then the `variant` salt. The plain rollout gate hashes the
91 same identifier with an *empty* salt, and mixing the two is the classic
92 reimplementation bug — so --selftest pins this string."""
93 return f " { flag_key } . { identifier } variant"
94
95
96 def pick_variant (h: float , variants: list[tuple[ str , float ]]) -> str | None :
97 """Walk the variants in stored order accumulating rollout_percentage / 100; the first
98 bound strictly above `h` wins. Mirrors the loop in get_matching_variant(). `variants`
99 must be in the flag's stored order — a wrong order inverts the result."""
100 cumulative = 0.0
101 for name, pct in variants:
102 cumulative += pct / 100.0
103 if h < cumulative:
104 return name
105 return None
106
107
108 def variant_for (flag_key: str , identifier: str , variants: list[tuple[ str , float ]]) -> str | None :
109 """Recompute the assigned variant, as get_matching_variant() would."""
110 return pick_variant(hash_of(variant_hash_key(flag_key, identifier)), variants)
111
112
113 # --- statistics -------------------------------------------------------------------------------
114 # Hand-rolled because the sandbox has no numpy/scipy (same constraint as ks2.py in the signals
115 # skills). Every function here is pinned in --selftest against published critical values.
116
117
118 def _gamma_p_series (s: float , x: float ) -> float :
119 """Regularized lower incomplete gamma P(s, x) by series expansion; converges for x < s + 1."""
120 term = 1.0 / s
121 total = term
122 for n in range ( 1 , 1000 ):
123 term *= x / (s + n)
124 total += term
125 if abs (term) < abs (total) * 1e-15 :
126 break
127 return total * math.exp( - x + s * math.log(x) - math.lgamma(s))
128
129
130 def _gamma_q_cf (s: float , x: float ) -> float :
131 """Regularized upper incomplete gamma Q(s, x) by Lentz continued fraction; for x >= s + 1."""
132 tiny = 1e-300
133 b = x + 1.0 - s
134 c = 1.0 / tiny
135 d = 1.0 / b
136 h = d
137 for i in range ( 1 , 1000 ):
138 an = - i * (i - s)
139 b += 2.0
140 d = an * d + b
141 if abs (d) < tiny:
142 d = tiny
143 c = b + an / c
144 if abs (c) < tiny:
145 c = tiny
146 d = 1.0 / d
147 delta = d * c
148 h *= delta
149 if abs (delta - 1.0 ) < 1e-15 :
150 break
151 return h * math.exp( - x + s * math.log(x) - math.lgamma(s))
152
153
154 def chi2_sf (x: float , dof: int ) -> float :
155 """P(chi-squared with `dof` d.o.f. > x) — the p-value for a goodness-of-fit statistic."""
156 if x <= 0 or dof < 1 :
157 return 1.0
158 s, scaled = dof / 2.0 , x / 2.0
159 return 1.0 - _gamma_p_series(s, scaled) if scaled < s + 1.0 else _gamma_q_cf(s, scaled)
160
161
162 @dataclass ( frozen = True )
163 class GoodnessOfFit :
164 """A chi-squared goodness-of-fit outcome: the statistic, its degrees of freedom, and the p."""
165
166 chi2: float
167 dof: int
168 p: float
169
170
171 @dataclass ( frozen = True )
172 class ConfidenceInterval :
173 """A two-sided interval on a proportion, as fractions in [0, 1]."""
174
175 low: float
176 high: float
177
178
179 def chi2_gof (observed: dict[ str , float ], expected: dict[ str , float ]) -> GoodnessOfFit:
180 """Goodness-of-fit of `observed` against `expected`."""
181 chi2 = sum ((observed.get(key, 0 ) - exp) ** 2 / exp for key, exp in expected.items() if exp > 0 )
182 dof = max ( len ([e for e in expected.values() if e > 0 ]) - 1 , 1 )
183 return GoodnessOfFit( chi2 = chi2, dof = dof, p = chi2_sf(chi2, dof))
184
185
186 def wilson_interval (k: int , n: int , z: float = 1.96 ) -> ConfidenceInterval:
187 """Wilson score interval for k successes in n trials. Beats the normal approximation at the
188 extremes, which is exactly where an agreement rate sits."""
189 if n <= 0 :
190 return ConfidenceInterval( low = 0.0 , high = 1.0 )
191 p = k / n
192 denom = 1.0 + z * z / n
193 center = p + z * z / ( 2 * n)
194 margin = z * math.sqrt(p * ( 1 - p) / n + z * z / ( 4 * n * n))
195 return ConfidenceInterval(
196 low = max ((center - margin) / denom, 0.0 ),
197 high = min ((center + margin) / denom, 1.0 ),
198 )
199
200
201 @dataclass ( frozen = True )
202 class VariantGap :
203 """The exact decomposition of one variant's deviation from the configured split."""
204
205 variant: str
206 expected: float
207 predicted: int
208 recorded: int
209
210 @ property
211 def gap (self) -> float :
212 """Observed minus configured — the SRM, as this sample sees it."""
213 return self .recorded - self .expected
214
215 @ property
216 def selection (self) -> float :
217 """How far the population that got recorded was already skewed. Capture-side."""
218 return self .predicted - self .expected
219
220 @ property
221 def reassignment (self) -> float :
222 """How many identifiers were recorded onto a different arm than the hash assigns."""
223 return self .recorded - self .predicted
224
225
226 def decompose (
227 recorded_counts: dict[ str , int ],
228 predicted_counts: dict[ str , int ],
229 variants: list[tuple[ str , float ]],
230 total: int ,
231 ) -> list[VariantGap]:
232 """Split each variant's gap into its selection and reassignment components."""
233 share_total = sum (pct for _, pct in variants) or 100.0
234 return [
235 VariantGap(
236 variant = name,
237 expected = total * (pct / share_total),
238 predicted = predicted_counts.get(name, 0 ),
239 recorded = recorded_counts.get(name, 0 ),
240 )
241 for name, pct in variants
242 ]
243
244
245 def chance_agreement (variants: list[tuple[ str , float ]]) -> float :
246 """Agreement a recompute would reach by luck alone if it carried no signal — sum of squared
247 variant shares. Hashing the wrong identifier (or a flag with experience continuity) lands
248 here, so an agreement rate that can't beat it means the test is inapplicable, not that
249 assignment is broken."""
250 share_total = sum (pct for _, pct in variants) or 100.0
251 return sum ((pct / share_total) ** 2 for _, pct in variants)
252
253
254 def printable (value: str ) -> str :
255 """Escape anything non-printable in a variant key before it reaches the report.
256
257 Variant keys are charset-validated in the PostHog UI but not by the API, and the recorded
258 values come from the customer's own CSV, so neither is trustworthy. Printed verbatim, a key
259 carrying a newline can forge a verdict line in a report an operator reads to pick a
260 diagnosis, and one carrying an escape sequence can drive their terminal. Printable
261 non-ASCII (a legitimately localized key) survives untouched."""
262 return "" .join(ch if ch.isprintable() else repr (ch)[ 1 : - 1 ] for ch in value)
263
264
265 def fmt_split (counts: dict[ str , float ], total: float ) -> str :
266 if total <= 0 :
267 return "(empty)"
268 return " " .join(
269 f " { printable(name) } = { counts.get(name, 0 ) :g} ( { 100.0 * counts.get(name, 0 ) / total :.1f} %)" for name in counts
270 )
271
272
273 # --- input ------------------------------------------------------------------------------------
274
275
276 def parse_variants (spec: str ) -> list[tuple[ str , float ]]:
277 out: list[tuple[ str , float ]] = []
278 for part in spec.split( "," ):
279 name, _, pct = part.partition( "=" )
280 if not name or not pct:
281 raise ValueError ( f "bad --variants entry { part !r} ; expected name=pct" )
282 try :
283 out.append((name.strip(), float (pct)))
284 except ValueError :
285 raise ValueError ( f "bad --variants entry { part !r} ; { pct !r} is not a number" ) from None
286 return out
287
288
289 def load_variants_file (path: str ) -> list[tuple[ str , float ]]:
290 """Read the flag's `filters.multivariate.variants` array straight from a file, preserving
291 stored order. Keeps variant keys off the command line — they are charset-validated only in
292 the UI, so a key arriving via the API can carry shell metacharacters or quotes.
293
294 Accepts the raw array, or the object that contains it (`multivariate`, or a whole flag)."""
295 with open (path) as fh:
296 blob = json.load(fh)
297 for step in ( "filters" , "multivariate" , "variants" ):
298 if isinstance (blob, dict ) and step in blob:
299 blob = blob[step]
300 if not isinstance (blob, list ) or not blob:
301 raise ValueError ( f " { path } : expected a non-empty variants array, got { type (blob). __name__ } " )
302 out: list[tuple[ str , float ]] = []
303 for entry in blob:
304 if not isinstance (entry, dict ) or "key" not in entry:
305 raise ValueError ( f " { path } : each variant needs a 'key', got { entry !r} " )
306 out.append(( str (entry[ "key" ]), float (entry.get( "rollout_percentage" , 0 ))))
307 return out
308
309
310 # --- verdict ----------------------------------------------------------------------------------
311
312
313 @dataclass ( frozen = True )
314 class Verdict :
315 label: str
316 lines: list[ str ]
317
318
319 def judge (
320 gaps: list[VariantGap],
321 agree: int ,
322 total: int ,
323 variants: list[tuple[ str , float ]],
324 ) -> Verdict:
325 """Route on the decomposition, not on a bare agreement threshold.
326
327 Order matters: the chance-agreement guard runs first, because a recompute that carries no
328 signal at all (wrong identifier, experience continuity) otherwise reads as a huge
329 assignment-side effect — the single most misleading failure this script can have."""
330 expected = {g.variant: g.expected for g in gaps}
331 recorded = {g.variant: float (g.recorded) for g in gaps}
332 predicted = {g.variant: float (g.predicted) for g in gaps}
333
334 recorded_fit = chi2_gof(recorded, expected)
335 predicted_fit = chi2_gof(predicted, expected)
336 agreement = wilson_interval(agree, total)
337 chance = chance_agreement(variants)
338
339 # The arm carrying the most gap is the one to decompose; its two shares sum to exactly 1.
340 lead = max (gaps, key =lambda g: abs (g.gap))
341 selection_share = lead.selection / lead.gap if lead.gap else 0.0
342 reassignment_share = lead.reassignment / lead.gap if lead.gap else 0.0
343
344 detail = [
345 f "lead arm: { printable(lead.variant) } (recorded { lead.recorded } vs expected { lead.expected :.1f} ,"
346 f " gap { lead.gap :+.1f} )" ,
347 f " selection: { lead.selection :+.1f} ( { 100.0 * selection_share :.0f} % of the gap)"
348 f " chi2= { predicted_fit.chi2 :.2f} p= { predicted_fit.p :.3g} " ,
349 f " reassignment: { lead.reassignment :+.1f} ( { 100.0 * reassignment_share :.0f} % of the gap)"
350 f " disagreement 95% CI [ { 100.0 * ( 1 - agreement.high) :.2f} %, { 100.0 * ( 1 - agreement.low) :.2f} %]" ,
351 ]
352
353 if agreement.low <= chance:
354 return Verdict(
355 "INAPPLICABLE" ,
356 detail
357 + [
358 "" ,
359 f "=> agreement { 100.0 * agree / total :.2f} % is not distinguishable from the"
360 f " { 100.0 * chance :.1f} % a coin" ,
361 " flip would reach on this split, so the recompute carries no signal. Almost always the" ,
362 " wrong identifier (group key? $device_id?) or ensure_experience_continuity = true." ,
363 " Fix the export or skip this test — do NOT read it as assignment-side." ,
364 ],
365 )
366
367 if recorded_fit.p > SRM_ALPHA :
368 return Verdict(
369 "NO SRM IN SAMPLE" ,
370 detail
371 + [
372 "" ,
373 f "=> the sample's own recorded split is consistent with the configured one"
374 f " (p= { recorded_fit.p :.3g} )." ,
375 " There is no gap here to localize. Either the sample is too small, the window is wrong," ,
376 " or the configured split you passed is not the one that was running." ,
377 ],
378 )
379
380 if selection_share >= DOMINANT_SHARE and predicted_fit.p < SRM_ALPHA :
381 return Verdict(
382 "CAPTURE" ,
383 detail
384 + [
385 "" ,
386 f "=> the users who got recorded were already skewed before assignment is considered:"
387 f " { 100.0 * selection_share :.0f} % of" ,
388 " the gap is selection. The skew is CAPTURE-side. Work the capture-side causes" ,
389 " (uneven-split exclusion, capture-by-surface, flag-read-before-load, wrong SDK method)." ,
390 ],
391 )
392
393 if reassignment_share >= DOMINANT_SHARE and agreement.high < 1.0 :
394 return Verdict(
395 "ASSIGNMENT" ,
396 detail
397 + [
398 "" ,
399 "=> the recorded variant disagrees with the hash often enough, and directionally enough,"
400 " to account for" ,
401 f " { 100.0 * reassignment_share :.0f} % of the gap. The skew is ASSIGNMENT-side."
402 " Work the assignment-side causes" ,
403 " (bootstrap inheritance, mid-run rehash, forced variant, stale local eval)." ,
404 " If disagreement clusters on one $lib/surface, start there." ,
405 ],
406 )
407
408 return Verdict(
409 "MIXED" ,
410 detail
411 + [
412 "" ,
413 f "=> neither component carries the gap on its own"
414 f " (selection { 100.0 * selection_share :.0f} %,"
415 f " reassignment { 100.0 * reassignment_share :.0f} %)." ,
416 " Work the larger one first, but do not present either as the single cause. A larger sample" ,
417 " (drop the LIMIT on the export query) is the cheapest way to separate them." ,
418 ],
419 )
420
421
422 # --- selftest ---------------------------------------------------------------------------------
423
424 # Golden vectors from rust/feature-flags/src/flags/flag_matching_utils.rs
425 # (test_calculate_hash: prefix="holdout-", salt=""). If these fail, the local
426 # hashing does not match PostHog and any verdict below would be meaningless.
427 # They cover the sha1 -> first-15-hex -> LONG_SCALE pipeline only.
428 _GOLDEN = [
429 ( "some_distinct_id" , 0.7270002403585725 ),
430 ( "test-identifier" , 0.4493881716040236 ),
431 ( "example_id" , 0.9402003475831224 ),
432 ( "example_id2" , 0.6292740389966519 ),
433 ]
434
435 # The variant path has no golden vector upstream — the Rust tests assert set membership
436 # (test_get_matching_variant_with_cache) and a +/-5pp distribution, never a fixed value.
437 # So pin the two things a reimplementation actually gets wrong, which the vectors above
438 # cannot see: the `{flag_key}.` prefix and the `variant` salt. A distribution check can't
439 # stand in for these — a wrong-but-deterministic hash still splits 50/50.
440 _GOLDEN_VARIANT_KEYS = [
441 ( "my-flag" , "user_1" , "my-flag.user_1variant" ),
442 ( "experiment-flag" , "some_distinct_id" , "experiment-flag.some_distinct_idvariant" ),
443 ]
444
445 # (hash, stored-order variants, expected) — covers the strict `<` bound, order
446 # sensitivity, and the sub-100% case that falls through to None.
447 _WALK_CASES : list[tuple[ float , list[tuple[ str , float ]], str | None ]] = [
448 ( 0.0 , [( "control" , 50.0 ), ( "test" , 50.0 )], "control" ),
449 ( 0.4999 , [( "control" , 50.0 ), ( "test" , 50.0 )], "control" ),
450 ( 0.5 , [( "control" , 50.0 ), ( "test" , 50.0 )], "test" ),
451 ( 0.9999 , [( "control" , 50.0 ), ( "test" , 50.0 )], "test" ),
452 ( 0.5 , [( "test" , 50.0 ), ( "control" , 50.0 )], "control" ),
453 ( 0.25 , [( "a" , 10.0 ), ( "b" , 30.0 ), ( "c" , 60.0 )], "b" ),
454 ( 0.95 , [( "a" , 10.0 ), ( "b" , 30.0 ), ( "c" , 60.0 )], "c" ),
455 ( 0.95 , [( "a" , 10.0 ), ( "b" , 30.0 )], None ),
456 ]
457
458 # Published upper-tail critical values: chi2_sf(x, dof) must return alpha.
459 _CHI2_CASES = [
460 ( 3.841459 , 1 , 0.05 ),
461 ( 10.827566 , 1 , 0.001 ),
462 ( 5.991465 , 2 , 0.05 ),
463 ( 13.815511 , 2 , 0.001 ),
464 ( 7.814728 , 3 , 0.05 ),
465 ( 16.266236 , 3 , 0.001 ),
466 ]
467
468 # The worked example in pulling-the-data.md: 832 vs 1123 pins down which split is running.
469 _SRM_EXAMPLE = [( 0.5 , 4.66e-11 ), ( 0.45 , 0.0299 ), ( 0.43 , 0.693 )]
470
471 # Untrusted variant keys reach the report from the API and from the customer's CSV. Each case is a
472 # forge attempt: a newline injecting a fake verdict line, and an ANSI sequence driving the terminal.
473 _PRINTABLE_CASES = [
474 ( "control" , "control" ),
475 ( "test \n => the skew is CAPTURE-side" , "test \\ n=> the skew is CAPTURE-side" ),
476 ( "test \x1b [2J" , "test \\ x1b[2J" ),
477 ( "control \t tab" , "control \\ ttab" ),
478 ( "variante_esp \u00e1 nol" , "variante_esp \u00e1 nol" ),
479 ]
480
481 _EVEN = [( "control" , 50.0 ), ( "test" , 50.0 )]
482
483 # (label, recorded, predicted, variants, agree, total, expected verdict).
484 # Pure capture: assignment is perfect (agreement 100%, predicted == recorded) but one arm's
485 # users were never recorded, so the served population is itself skewed.
486 # Pure assignment: the population is a clean 50/50 draw, but 120 identifiers were recorded
487 # onto the other arm.
488 _VERDICT_CASES : list[tuple[ str , dict[ str , int ], dict[ str , int ], list[tuple[ str , float ]], int , int , str ]] = [
489 ( "pure capture" , { "control" : 500 , "test" : 300 }, { "control" : 500 , "test" : 300 }, _EVEN , 800 , 800 , "CAPTURE" ),
490 ( "pure assignment" , { "control" : 520 , "test" : 280 }, { "control" : 400 , "test" : 400 }, _EVEN , 680 , 800 , "ASSIGNMENT" ),
491 # Greptile's case: 2% symmetric override noise beside a large capture skew. The old
492 # `pct >= 99.0` cutoff called this ASSIGNMENT-side purely because 98% < 99%.
493 ( "capture + 2% noise" , { "control" : 502 , "test" : 298 }, { "control" : 500 , "test" : 300 }, _EVEN , 784 , 800 , "CAPTURE" ),
494 # Balanced sample: nothing to localize.
495 ( "no srm" , { "control" : 400 , "test" : 400 }, { "control" : 400 , "test" : 400 }, _EVEN , 800 , 800 , "NO SRM IN SAMPLE" ),
496 # Wrong identifier: the recompute is uncorrelated, so agreement sits at the 50% chance rate.
497 ( "wrong identifier" , { "control" : 500 , "test" : 300 }, { "control" : 400 , "test" : 400 }, _EVEN , 400 , 800 , "INAPPLICABLE" ),
498 ]
499
500
501 def _check (ok: bool , label: str , got: object , want: object ) -> bool :
502 print ( f " { label :34s} { got !r} { 'ok' if ok else f 'MISMATCH (want { want !r} )' } " )
503 return ok
504
505
506 def selftest () -> int :
507 ok = True
508
509 print ( "hash pipeline (golden vectors from flag_matching_utils.rs):" )
510 for ident, expected in _GOLDEN :
511 got = calculate_hash( "holdout-" , ident, "" )
512 ok &= _check( abs (got - expected) < 1e-12 , ident, got, expected)
513
514 print ( "variant hash key (` {flag_key} .` prefix + `variant` salt):" )
515 for flag_key, ident, expected_key in _GOLDEN_VARIANT_KEYS :
516 got_key = variant_hash_key(flag_key, ident)
517 ok &= _check(got_key == expected_key, flag_key, got_key, expected_key)
518
519 print ( "variant walk (stored order, strict < bound):" )
520 for h, walk_variants, expected_variant in _WALK_CASES :
521 got_variant = pick_variant(h, walk_variants)
522 order = "," .join( f " { name } = { pct :g} " for name, pct in walk_variants)
523 ok &= _check(got_variant == expected_variant, f "h= { h :<7g} [ { order } ]" , got_variant, expected_variant)
524
525 print ( "chi-squared tail (published critical values):" )
526 for x, dof, alpha in _CHI2_CASES :
527 got_p = chi2_sf(x, dof)
528 ok &= _check( abs (got_p - alpha) < 1e-5 , f "chi2_sf( { x } , { dof } )" , round (got_p, 6 ), alpha)
529
530 print ( "chi-squared vs the worked example in pulling-the-data.md (832 vs 1123):" )
531 for share, expected_p in _SRM_EXAMPLE :
532 got_p = chi2_gof({ "a" : 832 , "b" : 1123 }, { "a" : 1955 * share, "b" : 1955 * ( 1 - share)}).p
533 rel = abs (got_p - expected_p) / expected_p
534 ok &= _check(rel < 0.01 , f "split { share :g} " , f " { got_p :.3g} " , f " { expected_p :.3g} " )
535
536 print ( "Wilson interval (the 99 % a greement that used to flip the verdict):" )
537 ci = wilson_interval( 792 , 800 )
538 lo, hi = ci.low, ci.high
539 bounds = ( round (lo, 4 ), round (hi, 4 ))
540 ok &= _check( abs (lo - 0.9804 ) < 1e-3 and abs (hi - 0.9949 ) < 1e-3 , "792/800" , bounds, ( 0.9804 , 0.9949 ))
541 ok &= _check(lo < 0.99 < hi, " straddles the old cutoff" , bounds, "0.99 inside" )
542
543 print ( "chance agreement (what a signal-free recompute reaches):" )
544 for spec, want_chance in (( _EVEN , 0.5 ), ([( "a" , 34.0 ), ( "b" , 33.0 ), ( "c" , 33.0 )], 0.3334 )):
545 got_chance = chance_agreement(spec)
546 ok &= _check( abs (got_chance - want_chance) < 1e-3 , f " { len (spec) } arms" , round (got_chance, 4 ), want_chance)
547
548 print ( "printable (untrusted variant keys cannot forge output):" )
549 for raw, want_out in _PRINTABLE_CASES :
550 got_out = printable(raw)
551 ok &= _check(got_out == want_out, repr (raw)[: 34 ], got_out, want_out)
552
553 print ( "verdict routing (synthetic samples):" )
554 for label, recorded, predicted, spec, agree, total, want in _VERDICT_CASES :
555 got_label = judge(decompose(recorded, predicted, spec, total), agree, total, spec).label
556 ok &= _check(got_label == want, label, got_label, want)
557
558 print ( "SELFTEST PASS" if ok else "SELFTEST FAILED" )
559 return 0 if ok else 1
560
561
562 # --- main -------------------------------------------------------------------------------------
563
564
565 def run (
566 flag_key: str ,
567 variants: list[tuple[ str , float ]],
568 csv_path: str ,
569 id_col: str ,
570 variant_col: str ,
571 variants_seen_col: str ,
572 include_ambiguous: bool ,
573 ) -> int :
574 total = agree = ambiguous = 0
575 recorded_counts: dict[ str , int ] = {}
576 predicted_counts: dict[ str , int ] = {}
577 with open (csv_path, newline = "" ) as fh:
578 reader = csv.DictReader(fh)
579 fields = reader.fieldnames or []
580 for col in (id_col, variant_col):
581 if col not in fields:
582 print ( f "error: column { col !r} not in CSV header { fields } " , file = sys.stderr)
583 return 2
584 has_seen_col = variants_seen_col in fields
585 for row in reader:
586 # An identifier that recorded more than one variant has no single "recorded" value to
587 # compare against, and collapsing it with argMin would hide the mid-run-rehash and
588 # bootstrap signatures outright. Count it, report it, keep it out of the rate.
589 if has_seen_col and not include_ambiguous:
590 try :
591 if float (row[variants_seen_col] or 1 ) > 1 :
592 ambiguous += 1
593 continue
594 except ValueError :
595 pass
596 predicted = variant_for(flag_key, row[id_col], variants)
597 recorded = row[variant_col]
598 total += 1
599 recorded_counts[recorded] = recorded_counts.get(recorded, 0 ) + 1
600 if predicted is not None :
601 predicted_counts[predicted] = predicted_counts.get(predicted, 0 ) + 1
602 if predicted == recorded:
603 agree += 1
604
605 if total == 0 :
606 print ( "error: no usable rows in CSV" , file = sys.stderr)
607 return 2
608
609 gaps = decompose(recorded_counts, predicted_counts, variants, total)
610 expected = {g.variant: g.expected for g in gaps}
611 agreement = wilson_interval(agree, total)
612
613 # A recorded value outside the configured keys can never agree with the hash, so it reads as
614 # total disagreement. Name it, or the verdict below sends the reader hunting for a wrong
615 # identifier when the real fault is the variant column (wrong property, or a stale key).
616 unknown = {v: n for v, n in recorded_counts.items() if v not in {g.variant for g in gaps}}
617
618 print ( f "rows: { total } " )
619 if ambiguous:
620 print ( f "ambiguous (skipped): { ambiguous } identifiers recorded >1 variant — see the note below" )
621 if not has_seen_col:
622 print ( f "note: no { variants_seen_col !r} column; re-export with it to surface rehashes" )
623 if unknown:
624 listed = ", " .join( f " { printable(v) } ( { n } )" for v, n in sorted (unknown.items(), key =lambda kv: - kv[ 1 ])[: 5 ])
625 print ( f "unknown variants: { sum (unknown.values()) } rows recorded a value not in --variants: { listed } " )
626 print (
627 f "agreement: { agree } / { total } ( { 100.0 * agree / total :.2f} %)"
628 f " 95% CI [ { 100.0 * agreement.low :.2f} %, { 100.0 * agreement.high :.2f} %]"
629 )
630 print ( f "configured split: { fmt_split(expected, float (total)) } " )
631 print ( f "predicted split: { fmt_split({k: float (v) for k, v in predicted_counts.items()}, float (total)) } " )
632 print ( f "recorded split: { fmt_split({k: float (v) for k, v in recorded_counts.items()}, float (total)) } " )
633 print ()
634
635 verdict = judge(gaps, agree, total, variants)
636 for line in verdict.lines:
637 print (line)
638 if ambiguous:
639 print ()
640 print ( f " Separately: { ambiguous } identifier(s) recorded more than one variant. Under 'first seen'" )
641 print ( " handling that is itself assignment-side evidence (mid-run rehash, bootstrap inheritance)." )
642 return 0
643
644
645 def main (argv: list[ str ]) -> int :
646 p = argparse.ArgumentParser( description = __doc__ , formatter_class = argparse.RawDescriptionHelpFormatter)
647 p.add_argument( "--selftest" , action = "store_true" , help = "replay golden vectors and statistics, then exit" )
648 p.add_argument( "--flag-key" , help = "feature flag key" )
649 p.add_argument(
650 "--variants-file" ,
651 help = "path to the flag's filters.multivariate.variants JSON (preferred: keeps untrusted "
652 "variant keys off the command line)" ,
653 )
654 p.add_argument( "--variants" , help = "stored-order variants, e.g. control=50,test=50" )
655 p.add_argument( "--csv" , help = "CSV export from the decisive-test query" )
656 p.add_argument( "--id-col" , default = "distinct_id" , help = "identifier column (default: distinct_id)" )
657 p.add_argument( "--variant-col" , default = "recorded_variant" , help = "recorded-variant column" )
658 p.add_argument( "--variants-seen-col" , default = "variants_seen" , help = "per-identifier variant-count column" )
659 p.add_argument(
660 "--include-ambiguous" ,
661 action = "store_true" ,
662 help = "count identifiers that recorded >1 variant in the agreement rate (default: report separately)" ,
663 )
664 args = p.parse_args(argv)
665
666 if args.selftest:
667 return selftest()
668 if args.variants_file and args.variants:
669 p.error( "pass --variants-file or --variants, not both" )
670 if not (args.flag_key and (args.variants_file or args.variants) and args.csv):
671 p.error( "--flag-key, --variants-file (or --variants) and --csv are required (or use --selftest)" )
672 try :
673 variants = load_variants_file(args.variants_file) if args.variants_file else parse_variants(args.variants)
674 except ( OSError , ValueError , json.JSONDecodeError) as e:
675 p.error( str (e))
676 return run(
677 args.flag_key,
678 variants,
679 args.csv,
680 args.id_col,
681 args.variant_col,
682 args.variants_seen_col,
683 args.include_ambiguous,
684 )
685
686
687 if __name__ == "__main__" :
688 raise SystemExit (main(sys.argv[ 1 :]))