1{2 "runs": {3 "with": 3,4 "base": 35 },6 "grading": "Blind LLM grader over cached runs, one assertion at a time. The grader states its reasoning in one sentence and then emits a VERDICT line. A grader constrained to a single-word answer with no reasoning returns unusable verdicts, so the reasoning step is required. Two graders are recorded because they disagree.",7 "overall": {8 "claude-opus-5": {9 "with_skill_run_pass_rate": 1.0,10 "baseline_run_pass_rate"
19 "blinding": "subject: fresh session per run with no conversation context; observer: automated runner; grader: separate session in a bare directory, given one output and one assertion, told neither the arm nor the expected classification",
20 "arms": "with_skill runs the skill as a forked subagent, so the graded artifact is the result the skill returns. baseline runs the same prompts in an environment where the skill does not exist, so the graded artifact is the assistant's own answer.",
21 "cases": [
22 {
23 "name": "conflict-throws-instead-of-result",
24 "with_skill": {
25 "runs": 3,
26 "run_pass_rate": {
27 "claude-opus-5": 1.0,
28 "claude-sonnet-5": 0.667
29 }
30 },
31 "baseline": {
32 "runs": 3,
33 "run_pass_rate": {
34 "claude-opus-5": 0.0,
35 "claude-sonnet-5": 0.0
36 }
37 },
38 "delta_run_pass_rate": {
39 "claude-opus-5": 1.0,
40 "claude-sonnet-5": 0.667
41 },
42 "per_assertion": [
43 {
44 "assertion": "Each finding is labeled with its type (conflict, gap, stale-reference, or aligned); how the label is rendered does not matter.",
45 "with_skill_pass_rate": {
46 "claude-opus-5": 1.0,
47 "claude-sonnet-5": 1.0
48 },
49 "baseline_pass_rate": {
50 "claude-opus-5": 0.333,
51 "claude-sonnet-5": 0.667
52 }
53 },
54 {
55 "assertion": "Output ends with a roll-up line giving counts per finding type or stating no in-force ADR was relevant.",
56 "with_skill_pass_rate": {
57 "claude-opus-5": 1.0,
58 "claude-sonnet-5": 0.667
59 },
60 "baseline_pass_rate": {
61 "claude-opus-5": 0.0,
62 "claude-sonnet-5": 0.0
63 }
64 },
65 {
66 "assertion": "Emits a CONFLICT finding.",
67 "with_skill_pass_rate": {
68 "claude-opus-5": 1.0,
69 "claude-sonnet-5": 1.0
70 },
71 "baseline_pass_rate": {
72 "claude-opus-5": 1.0,
73 "claude-sonnet-5": 1.0
74 }
75 },
76 {
77 "assertion": "Cites ADR-9001 (Result types for expected service errors) as the conflicting decision.",
78 "with_skill_pass_rate": {
79 "claude-opus-5": 1.0,
80 "claude-sonnet-5": 1.0
81 },
82 "baseline_pass_rate": {
83 "claude-opus-5": 1.0,
84 "claude-sonnet-5": 1.0
85 }
86 },
87 {
88 "assertion": "Does not classify this change as Aligned.",
89 "with_skill_pass_rate": {
90 "claude-opus-5": 1.0,
91 "claude-sonnet-5": 1.0
92 },
93 "baseline_pass_rate": {
94 "claude-opus-5": 1.0,
95 "claude-sonnet-5": 1.0
96 }
97 }
98 ]
99 },
100 {
101 "name": "gap-new-event-bus",
102 "with_skill": {
103 "runs": 3,
104 "run_pass_rate": {
105 "claude-opus-5": 1.0,
106 "claude-sonnet-5": 1.0
107 }
108 },
109 "baseline": {
110 "runs": 3,
111 "run_pass_rate": {
112 "claude-opus-5": 0.667,
113 "claude-sonnet-5": 1.0
114 }
115 },
116 "delta_run_pass_rate": {
117 "claude-opus-5": 0.333,
118 "claude-sonnet-5": 0.0
119 },
120 "per_assertion": [
121 {
122 "assertion": "Emits a GAP finding for the event bus decision.",
123 "with_skill_pass_rate": {
124 "claude-opus-5": 1.0,
125 "claude-sonnet-5": 1.0
126 },
127 "baseline_pass_rate": {
128 "claude-opus-5": 1.0,
129 "claude-sonnet-5": 1.0
130 }
131 },
132 {
133 "assertion": "Explains why it is significant (new cross-service pattern / precedent / blast radius).",
134 "with_skill_pass_rate": {
135 "claude-opus-5": 1.0,
136 "claude-sonnet-5": 1.0
137 },
138 "baseline_pass_rate": {
139 "claude-opus-5": 0.667,
140 "claude-sonnet-5": 1.0
141 }
142 },
143 {
144 "assertion": "Does not invent or cite a specific ADR number, title, or URL for the event bus.",
145 "with_skill_pass_rate": {
146 "claude-opus-5": 1.0,
147 "claude-sonnet-5": 1.0
148 },
149 "baseline_pass_rate": {
150 "claude-opus-5": 1.0,
151 "claude-sonnet-5": 1.0
152 }
153 }
154 ]
155 },
156 {
157 "name": "aligned-returns-result",
158 "with_skill": {
159 "runs": 3,
160 "run_pass_rate": {
161 "claude-opus-5": 1.0,
162 "claude-sonnet-5": 1.0
163 }
164 },
165 "baseline": {
166 "runs": 3,
167 "run_pass_rate": {
168 "claude-opus-5": 0.667,
169 "claude-sonnet-5": 0.333
170 }
171 },
172 "delta_run_pass_rate": {
173 "claude-opus-5": 0.333,
174 "claude-sonnet-5": 0.667
175 },
176 "per_assertion": [
177 {
178 "assertion": "Classifies the change as Aligned with ADR-9001 (or otherwise reports no conflict/gap).",
179 "with_skill_pass_rate": {
180 "claude-opus-5": 1.0,
181 "claude-sonnet-5": 1.0
182 },
183 "baseline_pass_rate": {
184 "claude-opus-5": 1.0,
185 "claude-sonnet-5": 1.0
186 }
187 },
188 {
189 "assertion": "Does not emit a CONFLICT or GAP finding for this change.",
190 "with_skill_pass_rate": {
191 "claude-opus-5": 1.0,
192 "claude-sonnet-5": 1.0
193 },
194 "baseline_pass_rate": {
195 "claude-opus-5": 1.0,
196 "claude-sonnet-5": 0.667
197 }
198 },
199 {
200 "assertion": "Keeps the aligned note brief (does not manufacture findings).",
201 "with_skill_pass_rate": {
202 "claude-opus-5": 1.0,
203 "claude-sonnet-5": 1.0
204 },
205 "baseline_pass_rate": {
206 "claude-opus-5": 0.667,
207 "claude-sonnet-5": 0.333
208 }
209 }
210 ]
211 },
212 {
213 "name": "stale-reference-superseded-adr",
214 "with_skill": {
215 "runs": 3,
216 "run_pass_rate": {
217 "claude-opus-5": 1.0,
218 "claude-sonnet-5": 1.0
219 }
220 },
221 "baseline": {
222 "runs": 3,
223 "run_pass_rate": {
224 "claude-opus-5": 1.0,
225 "claude-sonnet-5": 1.0
226 }
227 },
228 "delta_run_pass_rate": {
229 "claude-opus-5": 0.0,
230 "claude-sonnet-5": 0.0
231 },
232 "per_assertion": [
233 {
234 "assertion": "Emits a STALE-REFERENCE finding (or flags that ADR-9002 is superseded).",
235 "with_skill_pass_rate": {
236 "claude-opus-5": 1.0,
237 "claude-sonnet-5": 1.0
238 },
239 "baseline_pass_rate": {
240 "claude-opus-5": 1.0,
241 "claude-sonnet-5": 1.0
242 }
243 },
244 {
245 "assertion": "Points to ADR-9005 (Jest) as the in-force decision.",
246 "with_skill_pass_rate": {
247 "claude-opus-5": 1.0,
248 "claude-sonnet-5": 1.0
249 },
250 "baseline_pass_rate": {
251 "claude-opus-5": 1.0,
252 "claude-sonnet-5": 1.0
253 }
254 },
255 {
256 "assertion": "Does not treat ADR-9002 (Vitest) as a current, binding constraint.",
257 "with_skill_pass_rate": {
258 "claude-opus-5": 1.0,
259 "claude-sonnet-5": 1.0
260 },
261 "baseline_pass_rate": {
262 "claude-opus-5": 1.0,
263 "claude-sonnet-5": 1.0
264 }
265 }
266 ]
267 },
268 {
269 "name": "skip-no-architectural-surface",
270 "with_skill": {
271 "runs": 3,
272 "run_pass_rate": {
273 "claude-opus-5": 1.0,
274 "claude-sonnet-5": 1.0
275 }
276 },
277 "baseline": {
278 "runs": 3,
279 "run_pass_rate": {
280 "claude-opus-5": 0.667,
281 "claude-sonnet-5": 0.667
282 }
283 },
284 "delta_run_pass_rate": {
285 "claude-opus-5": 0.333,
286 "claude-sonnet-5": 0.333
287 },
288 "per_assertion": [
289 {
290 "assertion": "Reports no relevant in-force ADR / no architectural surface.",
291 "with_skill_pass_rate": {
292 "claude-opus-5": 1.0,
293 "claude-sonnet-5": 1.0
294 },
295 "baseline_pass_rate": {
296 "claude-opus-5": 0.667,
297 "claude-sonnet-5": 0.667
298 }
299 },
300 {
301 "assertion": "Does not manufacture a CONFLICT, GAP, or STALE-REFERENCE finding.",
302 "with_skill_pass_rate": {
303 "claude-opus-5": 1.0,
304 "claude-sonnet-5": 1.0
305 },
306 "baseline_pass_rate": {
307 "claude-opus-5": 1.0,
308 "claude-sonnet-5": 1.0
309 }
310 },
311 {
312 "assertion": "Does not invent an ADR.",
313 "with_skill_pass_rate": {
314 "claude-opus-5": 1.0,
315 "claude-sonnet-5": 1.0
316 },
317 "baseline_pass_rate": {
318 "claude-opus-5": 1.0,
319 "claude-sonnet-5": 1.0
320 }
321 }
322 ]
323 },
324 {
325 "name": "no-adr-found-do-not-invent",
326 "with_skill": {
327 "runs": 3,
328 "run_pass_rate": {
329 "claude-opus-5": 1.0,
330 "claude-sonnet-5": 1.0
331 }
332 },
333 "baseline": {
334 "runs": 3,
335 "run_pass_rate": {
336 "claude-opus-5": 1.0,
337 "claude-sonnet-5": 1.0
338 }
339 },
340 "delta_run_pass_rate": {
341 "claude-opus-5": 0.0,
342 "claude-sonnet-5": 0.0
343 },
344 "per_assertion": [
345 {
346 "assertion": "Reports that no in-force ADR governs the change (none found).",
347 "with_skill_pass_rate": {
348 "claude-opus-5": 1.0,
349 "claude-sonnet-5": 1.0
350 },
351 "baseline_pass_rate": {
352 "claude-opus-5": 1.0,
353 "claude-sonnet-5": 1.0
354 }
355 },
356 {
357 "assertion": "Does not invent an ADR number, title, or URL.",
358 "with_skill_pass_rate": {
359 "claude-opus-5": 1.0,
360 "claude-sonnet-5": 1.0
361 },
362 "baseline_pass_rate": {
363 "claude-opus-5": 1.0,
364 "claude-sonnet-5": 1.0
365 }
366 },
367 {
368 "assertion": "Does not fabricate a conflict.",