Eval

1{
2  "$comment": "schema_version: 1; contract source: agents/llm2bedrock-prompt-evaluator.md §14 — live_source_baseline / judge_model / source_baseline_quality are marked MANDATORY there (the old JS schema omitted them from required); notes carries the structured prefix lines. partial exists ONLY in this schema (the evaluator is the sole agent documented to emit it). blocked reasons per evaluator §14.",
3  "oneOf": [
4    {
5      "type": "object",
6      "additionalProperties": false,
7      "required": [
8        "eval_report_path",
9        "pass_rate",
10        "total_cases",
11        "failures",
12        "notes",
13        "live_source_baseline",
14        "judge_model",
15        "source_baseline_quality"
16      ],
17      "properties": {
18        "summary": { "type": "string" },
19        "eval_report_path": { "type": "string" },
20        "pass_rate": { "type": "number", "minimum": 0, "maximum": 1 },
21        "total_cases": { "type": "integer", "minimum": 0 },
22        "failures": { "type": "integer", "minimum": 0 },
23        "notes": { "type": "string" },
24        "live_source_baseline": { "type": "boolean" },
25        "judge_model": { "type": "string" },
26        "source_baseline_quality": { "type": "string", "enum": ["good", "poor", "unknown"] }
27      }
28    },
29    {
30      "type": "object",
31      "additionalProperties": false,
32      "required": ["blocked"],
33      "properties": {
34        "blocked": {
35          "type": "object",
36          "additionalProperties": false,
37          "required": ["reason", "detail"],
38          "properties": {
39            "reason": {
40              "type": "string",
41              "enum": ["model_access", "source_key_auth", "model_unresolvable", "authz", "mantle_deps_missing"]
42            },
43            "detail": { "type": "string" }
44          }
45        }
46      }
47    },
48    {
49      "type": "object",
50      "additionalProperties": false,
51      "required": ["partial"],
52      "properties": {
53        "partial": {
54          "type": "object",
55          "additionalProperties": false,
56          "required": ["completed", "total", "reason"],
57          "properties": {
58            "completed": { "type": "integer", "minimum": 0 },
59            "total": { "type": "integer", "minimum": 0 },
60            "reason": { "type": "string", "enum": ["throttled"] }
61          }
62        }
63      }
64    }
65  ]
66}