Aggregate Catch Rate

1#!/usr/bin/env bash
2# Combines every result.json under OUT_DIR into one benchmark document on stdout.
3# Usage: aggregate-catch-rate.sh <model> <grader-model> > catch-rate-baseline.json
4# Env:   OUT_DIR  same scratch directory run-catch-rate.sh and score-catch-rate.sh used
5set -euo pipefail
6
7: "${OUT_DIR:?set OUT_DIR}"
8
9find "$OUT_DIR" -name result.json -print0 | xargs -0 cat | jq -s \
10  --arg model "$1" --arg grader "$2" --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '
11  def ratio(a; b): (b // 0) as $b | if $b == 0 then null else (((a // 0) / $b * 1000) | round / 1000) end;
12  # An errored run measured nothing, so it is left out of every rate rather than scored as a miss.
13  def summary:
14    ([.[] | select(.error)] | length) as $errored
15    | [.[] | select(.error | not)]
16    | group_by(.run_number) as $by_run
17    | {
18        runs: length,
19        catch_rate_per_run: [$by_run[] | {run_number: .[0].run_number, catch_rate: ratio([.[] | select(.hit)] | length; length)}],
20        mean_catch_rate: ratio([.[] | select(.hit)] | length; length),
21        false_finding_rate: ratio([.[].false_findings] | add; [.[].validated_findings] | add),
22        validated_findings: ([.[].validated_findings] | add),
23        false_findings: ([.[].false_findings] | add),
24        errored_runs: $errored
25      };
26  sort_by(.eval_id, .configuration, .run_number) as $runs
27  | {
28      metadata: {
29        skill_name: "performing-multi-agent-code-review",
30        eval_type: "catch-rate",
31        executor_model: $model,
32        grader_model: $grader,
33        timestamp: $ts,
34        evals_run: [$runs[].eval_id] | unique,
35        runs_per_configuration: ([$runs[].run_number] | max)
36      },
37      run_summary: ($runs | group_by(.configuration) | map({key: .[0].configuration, value: summary}) | from_entries),
38      runs: $runs
39    }'