Behavior Baseline

1{
2  "grader_model": "claude-opus-5",
3  "denominator": "expectations x runs_per_arm",
4  "per_expectation": "passes per expectation, in behavior-eval.json order, out of runs_per_arm",
5  "claude-sonnet-5/default": {
6    "default-is-main-no-hint": {
7      "runs_per_arm": 4,
8      "new-skill": "16/16",
9      "old-skill": "12/16",
10      "no-skill": "12/16",
11      "new-skill-per-expectation": "4/4/4/4",
12      "old-skill-per-expectation": "0/4/4/4",
13      "no-skill-per-expectation": "0/4/4/4"
14    },
15    "already-off-default": {
16      "runs_per_arm": 2,
17      "new-skill": "6/6",
18      "old-skill": "6/6",
19      "no-skill": "6/6"
20    },
21    "user-supplies-branch-name": {
22      "runs_per_arm": 2,
23      "new-skill": "8/8",
24      "old-skill": "8/8",
25      "no-skill": "8/8"
26    },
27    "user-delegates-branch-naming": {
28      "runs_per_arm": 2,
29      "new-skill": "8/8",
30      "old-skill": "8/8",
31      "no-skill": "8/8"
32    },
33    "default-is-master": {
34      "runs_per_arm": 4,
35      "new-skill": "16/16",
36      "old-skill": "16/16",
37      "no-skill": "14/16",
38      "new-skill-per-expectation": "4/4/4/4",
39      "old-skill-per-expectation": "4/4/4/4",
40      "no-skill-per-expectation": "2/4/4/4"
41    },
42    "default-unresolvable": {
43      "runs_per_arm": 4,
44      "new-skill": "15/16",
45      "old-skill": "4/16",
46      "no-skill": "4/16",
47      "new-skill-per-expectation": "4/4/3/4",
48      "old-skill-per-expectation": "0/0/0/4",
49      "no-skill-per-expectation": "0/0/0/4"
50    }
51  }
52}