Setting the file. One moment.
Scoring · Agent Observability Build Eval From Annotations · datadog-labs/agent-skills · Skills Docs
Back to the top of the page def class_support
— line 164
This file
Number 2.3
Position 3 of 3
Type Python
Size 17 KB
Lines 393 references/ scoring.py
Python · 393 lines · 17 KB
"""
15
16 from __future__ import annotations
17
18 import argparse
19 import json
20 import math
21 import sys
22 from collections import Counter, defaultdict
23 from pathlib import Path
24
25
26 # --------------------------------------------------------------------------- helpers
27
28
29 def wilson (successes: int , total: int , z: float = 1.96 ):
30 """Wilson 95% interval — the honest one at n=13, unlike the normal approximation."""
31 if total == 0 :
32 return ( 0.0 , 0.0 )
33 p = successes / total
34 denom = 1 + z * z / total
35 centre = (p + z * z / ( 2 * total)) / denom
36 half = z * math.sqrt(p * ( 1 - p) / total + z * z / ( 4 * total * total)) / denom
37 return ( round ( max ( 0.0 , centre - half), 4 ), round ( min ( 1.0 , centre + half), 4 ))
38
39
40 def mcnemar_exact (b: int , c: int ) -> float :
41 """Two-sided exact binomial p on the discordant pairs. b/c = rows only one version got right."""
42 n = b + c
43 if n == 0 :
44 return 1.0
45 k = min (b, c)
46 tail = sum (math.comb(n, i) for i in range (k + 1 )) / ( 2 ** n)
47 return round ( min ( 1.0 , 2 * tail), 4 )
48
49
50 # --------------------------------------------------------------------------- label identity
51
52 # A categorical label value arrives from the annotation API as a LIST, even when only one value was
53 # chosen (`["permanent"]`), and a multi-select row carries several. Truth and prediction therefore
54 # have to be compared as sets, not as JSON text: `["a","b"]` and `["b","a"]` are the same answer.
55
56
57 def _atom (value):
58 """One class, hashable. Scalars stay themselves so the report reads in the user's own words."""
59 return value if isinstance (value, ( str , int , float , bool )) or value is None \
60 else json.dumps(value, sort_keys = True )
61
62
63 def as_set (value) -> frozenset :
64 """Any label value -> the set of classes it names. A scalar is a set of one."""
65 if isinstance (value, ( list , tuple , set , frozenset )):
66 return frozenset (_atom(v) for v in value)
67 return frozenset ([_atom(value)])
68
69
70 def key (value) -> str :
71 """Canonical, order-insensitive, READABLE identity of a label value.
72
73 Readable matters: this string is what lands in the confusion matrix and the per-class recall of
74 the final report, where a human has to recognise their own classes in it.
75 """
76 parts = sorted ( str (a) for a in as_set(value))
77 return "+" .join(parts) if parts else "<empty>"
78
79
80 def make_credit (mode: str , groups = None , partial: float = 0.5 ):
81 """-> credit(truth, pred) in [0,1]. How much of a right answer a prediction is.
82
83 `exact` is all-or-nothing and is the only honest default. The graded modes exist because some
84 label sets have near-misses that are genuinely worth more than a wrong answer — but the grading
85 has to come from the user's own taxonomy, never from the judge's opinion of its own answer:
86
87 * `jaccard` — overlap over union, for multi-select labels where getting 1 of 2 right is
88 partial work done;
89 * `similarity_group` — `partial` credit when truth and prediction fall in the same
90 user-supplied group of classes, mirroring an experiment's own similarity metric.
91 """
92 lookup = {}
93 for index, group in enumerate (groups or []):
94 for member in group:
95 lookup[_atom(member)] = index
96
97 def credit (truth, pred) -> float :
98 t, p = as_set(truth), as_set(pred)
99 if t == p:
100 return 1.0
101 if mode == "exact" :
102 return 0.0
103 if mode == "jaccard" :
104 return len (t & p) / len (t | p) if (t | p) else 0.0
105 if mode == "similarity_group" :
106 groups_t = {lookup[m] for m in t if m in lookup}
107 groups_p = {lookup[m] for m in p if m in lookup}
108 return partial if groups_t and groups_t == groups_p else 0.0
109 raise SystemExit ( f "unknown match mode { mode !r} " )
110
111 return credit
112
113
114 def majority (values: list ):
115 """Majority vote over the runs. Ties (possible only with an even --runs) return None.
116
117 Grouped by canonical key, so two passes that answered the same multi-select in a different
118 order count as agreeing rather than as a 1-1 tie.
119 """
120 if not values:
121 return None
122 counts = Counter(key(v) for v in values).most_common()
123 if len (counts) > 1 and counts[ 0 ][ 1 ] == counts[ 1 ][ 1 ]:
124 return None
125 winner = counts[ 0 ][ 0 ]
126 return next (v for v in values if key(v) == winner)
127
128
129 _MISSING = object ()
130
131
132 def pick (value, field):
133 """One label out of a joint verdict. `field=None` means the verdict IS the label."""
134 if field is None :
135 return value
136 if not isinstance (value, dict ):
137 raise SystemExit (
138 f "--label-field { field !r} needs a verdict whose label is an object of "
139 f "label-name -> value; got { type (value). __name__ } ."
140 )
141 return value.get(field, _MISSING )
142
143
144 # --------------------------------------------------------------------------- metrics
145
146
147 def confusion (pairs):
148 """pairs = [(truth, pred)]. Returns {(truth, pred): count} over whatever classes exist."""
149 matrix = defaultdict( int )
150 for truth, pred in pairs:
151 matrix[(key(truth), key(pred))] += 1
152 return matrix
153
154
155 def per_class_recall (pairs):
156 hit, total = defaultdict( int ), defaultdict( int )
157 for truth, pred in pairs:
158 cls = key(truth)
159 total[ cls ] += 1
160 hit[ cls ] += int (key(truth) == key(pred))
161 return {k: round (hit[k] / total[k], 4 ) for k in total}
162
163
164 def class_support (pairs):
165 """Rows per truth class. A recall computed on 1 row is a coin toss with a decimal point."""
166 return dict (Counter(key(t) for t, _ in pairs))
167
168
169 def score (pairs, metric: str , credit = None ) -> float :
170 if not pairs:
171 return 0.0
172 credit = credit or make_credit( "exact" )
173 if metric == "accuracy" :
174 return round ( sum (key(t) == key(p) for t, p in pairs) / len (pairs), 4 )
175 if metric == "mean_credit" :
176 # the graded headline: partial answers score partially. Only meaningful with a --match
177 # mode other than exact, where it is identical to accuracy.
178 return round ( sum (credit(t, p) for t, p in pairs) / len (pairs), 4 )
179 if metric == "balanced_accuracy" :
180 recalls = per_class_recall(pairs)
181 return round ( sum (recalls.values()) / len (recalls), 4 )
182 if metric in ( "f1" , "f1_minority" ):
183 classes = Counter(key(t) for t, _ in pairs)
184 if len (classes) > 2 :
185 raise SystemExit (
186 f " { metric !r} needs a two-class corpus; this one has { len (classes) } classes, so the "
187 "'minority class' is whichever class happens to be rarest and the score says "
188 "nothing about the rest. Use macro_f1, cohens_kappa or mean_credit."
189 )
190 positive = min (classes, key = classes.get) # minority class is the positive one
191 tp = sum (key(t) == positive and key(p) == positive for t, p in pairs)
192 fp = sum (key(t) != positive and key(p) == positive for t, p in pairs)
193 fn = sum (key(t) == positive and key(p) != positive for t, p in pairs)
194 if tp == 0 :
195 return 0.0
196 precision, recall = tp / (tp + fp), tp / (tp + fn)
197 return round ( 2 * precision * recall / (precision + recall), 4 )
198 if metric == "macro_f1" :
199 classes = {key(t) for t, _ in pairs}
200 scores = []
201 for cls in classes:
202 tp = sum (key(t) == cls and key(p) == cls for t, p in pairs)
203 fp = sum (key(t) != cls and key(p) == cls for t, p in pairs)
204 fn = sum (key(t) == cls and key(p) != cls for t, p in pairs)
205 scores.append( 0.0 if tp == 0 else 2 * tp / ( 2 * tp + fp + fn))
206 return round ( sum (scores) / len (scores), 4 )
207 if metric == "cohens_kappa" :
208 observed = sum (key(t) == key(p) for t, p in pairs) / len (pairs)
209 truths = Counter(key(t) for t, _ in pairs)
210 preds = Counter(key(p) for _, p in pairs)
211 expected = sum (truths[c] * preds.get(c, 0 ) for c in truths) / ( len (pairs) ** 2 )
212 return round ( 0.0 if expected == 1 else (observed - expected) / ( 1 - expected), 4 )
213 if metric == "mae" :
214 return round ( sum ( abs ( float (t) - float (p)) for t, p in pairs) / len (pairs), 4 )
215 raise SystemExit ( f "unknown metric { metric !r} " )
216
217
218 def constant_baseline (pairs, metric: str , credit = None ) -> float :
219 """What the laziest possible judge scores. The headline must beat this to mean anything."""
220 seen, candidates = set (), []
221 for truth, _ in pairs: # one candidate per distinct class, keeping the original value shape
222 if key(truth) not in seen:
223 seen.add(key(truth))
224 candidates.append(truth)
225 return max (score([(t, cand) for t, _ in pairs], metric, credit) for cand in candidates)
226
227
228 # --------------------------------------------------------------------------- load
229
230
231 def load_predictions (path: str , label_field = None ):
232 """-> {row_id: {"vote": label|None, "flipped": bool, "usable": bool, "confidence": int|None}}
233
234 Confidence is carried through and reported, never used to weigh the vote: the prediction is the
235 majority label and nothing else (rubric §6). A row's confidence is the mean of the passes that
236 reported a usable one; ``None`` when no pass did.
237 """
238 passes = defaultdict( list )
239 confidences = defaultdict( list )
240 unparseable = defaultdict( int )
241 no_confidence = defaultdict( int )
242 for line in Path(path).read_text().splitlines():
243 if not line.strip():
244 continue
245 rec = json.loads(line)
246 if rec.get( "unparseable" ) or "label" not in rec:
247 unparseable[rec[ "id" ]] += 1
248 continue
249 label = pick(rec[ "label" ], label_field)
250 if label is _MISSING : # the pass answered, but not for this label of a joint verdict
251 unparseable[rec[ "id" ]] += 1
252 continue
253 passes[rec[ "id" ]].append(label)
254 if isinstance (rec.get( "confidence" ), ( int , float )) and not isinstance (rec.get( "confidence" ), bool ):
255 confidences[rec[ "id" ]].append(rec[ "confidence" ])
256 else :
257 no_confidence[rec[ "id" ]] += 1
258 out = {}
259 for row_id in set (passes) | set (unparseable):
260 labels = passes.get(row_id, [])
261 vote = majority(labels)
262 seen = confidences.get(row_id, [])
263 out[row_id] = {
264 "vote" : vote,
265 "flipped" : len ({json.dumps(v) for v in labels}) > 1 ,
266 "usable" : vote is not None ,
267 "unparseable_passes" : unparseable.get(row_id, 0 ),
268 "confidence" : round ( sum (seen) / len (seen), 1 ) if seen else None ,
269 "passes_without_confidence" : no_confidence.get(row_id, 0 ),
270 }
271 return out
272
273
274 def calibration (scored, preds):
275 """Is the judge's stated confidence worth anything? Compare it against being right.
276
277 A judge equally confident when wrong as when right is telling the user nothing, and that is a
278 reportable fact about the judge — not a reason to change the score.
279 """
280 with_conf = [(preds[rid][ "confidence" ], key(t) == key(p)) for rid, t, p in scored
281 if preds[rid][ "confidence" ] is not None ]
282 missing = sum ( 1 for rid, _, _ in scored if preds[rid][ "confidence" ] is None )
283 if not with_conf:
284 return { "rows_with_confidence" : 0 , "rows_without_confidence" : missing}
285 right = [c for c, ok in with_conf if ok]
286 wrong = [c for c, ok in with_conf if not ok]
287 bands = {}
288 for lo, hi in (( 0 , 59 ), ( 60 , 79 ), ( 80 , 89 ), ( 90 , 100 )):
289 band = [ok for c, ok in with_conf if lo <= c <= hi]
290 if band:
291 bands[ f " { lo } - { hi } %" ] = { "rows" : len (band), "accuracy" : round ( sum (band) / len (band), 4 )}
292 return {
293 "rows_with_confidence" : len (with_conf),
294 "rows_without_confidence" : missing,
295 "mean_confidence" : round ( sum (c for c, _ in with_conf) / len (with_conf), 1 ),
296 "mean_confidence_when_right" : round ( sum (right) / len (right), 1 ) if right else None ,
297 "mean_confidence_when_wrong" : round ( sum (wrong) / len (wrong), 1 ) if wrong else None ,
298 "accuracy_by_confidence_band" : bands,
299 }
300
301
302 def main () -> None :
303 ap = argparse.ArgumentParser()
304 ap.add_argument( "--corpus" , required = True )
305 ap.add_argument( "--pred" , required = True )
306 ap.add_argument( "--metric" , default = "balanced_accuracy" )
307 ap.add_argument( "--split" , default = "train" , choices = [ "train" , "holdout" , "all" ])
308 ap.add_argument( "--baseline-pred" , help = "predictions of the current best, for McNemar" )
309 ap.add_argument( "--label-field" , help = "for a joint judge: which label of the verdict to score" )
310 ap.add_argument( "--match" , default = "exact" , choices = [ "exact" , "jaccard" , "similarity_group" ],
311 help = "how much credit a partly-right answer gets (default: none)" )
312 ap.add_argument( "--groups" , help = "JSON file: { \" groups \" : [[classA, classB], ...], "
313 " \" partial_credit \" : 0.5} for --match similarity_group" )
314 ap.add_argument( "--small-class-floor" , type = int , default = 6 ,
315 help = "truth classes with fewer rows than this are reported as not measurable" )
316 args = ap.parse_args()
317
318 groups, partial = None , 0.5
319 if args.groups:
320 spec = json.loads(Path(args.groups).read_text())
321 groups, partial = spec.get( "groups" , []), spec.get( "partial_credit" , 0.5 )
322 if args.match == "similarity_group" and not groups:
323 sys.exit( "--match similarity_group needs --groups: the taxonomy is the user's, not the judge's." )
324 credit = make_credit(args.match, groups, partial)
325
326 rows = [json.loads(l) for l in Path(args.corpus).read_text().splitlines() if l.strip()]
327 if args.split != "all" :
328 rows = [r for r in rows if r.get( "split" , "train" ) == args.split]
329 truth = {}
330 for r in rows:
331 value = pick(r[ "label" ], args.label_field)
332 if value is _MISSING :
333 sys.exit( f "corpus row { r[ 'id' ] } has no label { args.label_field !r} ." )
334 truth[r[ "id" ]] = value
335
336 preds = load_predictions(args.pred, args.label_field)
337 scored = [(rid, truth[rid], preds[rid][ "vote" ]) for rid in truth if preds.get(rid, {}).get( "usable" )]
338 excluded = [rid for rid in truth if not preds.get(rid, {}).get( "usable" )]
339 pairs = [(t, p) for _, t, p in scored]
340
341 correct = {rid: (key(t) == key(p)) for rid, t, p in scored}
342 headline = score(pairs, args.metric, credit)
343 hits = sum (correct.values())
344 support = class_support(pairs)
345 small = {c: n for c, n in support.items() if n < args.small_class_floor}
346
347 result = {
348 "metric" : args.metric,
349 "label_field" : args.label_field,
350 "match_mode" : args.match,
351 "split" : args.split,
352 "headline" : headline,
353 "rows_scored" : len (pairs),
354 "rows_excluded_unusable" : len (excluded),
355 "excluded_ids" : excluded,
356 "raw_accuracy" : round (hits / len (pairs), 4 ) if pairs else 0.0 ,
357 "wilson_95_on_accuracy" : wilson(hits, len (pairs)),
358 "constant_class_baseline" : constant_baseline(pairs, args.metric, credit) if pairs else 0.0 ,
359 "mean_credit" : score(pairs, "mean_credit" , credit),
360 "per_class_recall" : per_class_recall(pairs),
361 "class_support" : support,
362 # recall on 3 rows is not a measurement; the report must say so rather than quoting it
363 "classes_below_floor" : small,
364 "confusion" : { f "truth= { t } |pred= { p } " : n for (t, p), n in confusion(pairs).items()},
365 "flip_rate" : round ( sum (preds[rid][ "flipped" ] for rid, _, _ in scored) / len (scored), 4 ) if scored else 0.0 ,
366 "correct_by_row" : correct,
367 "confidence" : calibration(scored, preds),
368 }
369
370 if args.baseline_pred:
371 base = load_predictions(args.baseline_pred)
372 b = c = 0 # b: only candidate right, c: only baseline right
373 for rid, t, p in scored:
374 if not base.get(rid, {}).get( "usable" ):
375 continue
376 base_right, cand_right = key(base[rid][ "vote" ]) == key(t), key(p) == key(t)
377 b += int (cand_right and not base_right)
378 c += int (base_right and not cand_right)
379 result[ "vs_baseline" ] = {
380 "gained" : b,
381 "lost" : c,
382 "mcnemar_p" : mcnemar_exact(b, c),
383 "gained_ids" : [rid for rid, t, p in scored if base.get(rid, {}).get( "usable" )
384 and key(p) == key(t) and key(base[rid][ "vote" ]) != key(t)],
385 "lost_ids" : [rid for rid, t, p in scored if base.get(rid, {}).get( "usable" )
386 and key(p) != key(t) and key(base[rid][ "vote" ]) == key(t)],
387 }
388
389 print (json.dumps(result, indent = 2 ))
390
391
392 if __name__ == "__main__" :
393 main()