Skill 12 · Performing Multi Agent Code Review

Subchapter 12.15

Run Catch Rate

Catch Rate Baseline

Catch Rate Eval

gh

Setup

4 more in the appendix

Run Catch Rate

Perform Preflight

Running Work Transitions

Threat Modeling

Triaging Security Findings

Raw file

bitwarden/ai-plugins

evals/run-catch-rate.sh

Shell89 lines5 KB

1#!/usr/bin/env bash
2# Runs one catch-rate eval case: one corpus PR, one configuration, one run number.
3# Usage: run-catch-rate.sh <eval-id> <with_skill|without_skill> <run-number>
4# Env:   CLONES_DIR  directory holding a local clone of each corpus repo, named by repo (server, clients, ...)
5#        OUT_DIR     scratch directory for checkouts, transcripts, and reports
6#        MODEL       optional, defaults to opus
7set -euo pipefail
8
9EVALS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
10PLUGINS_DIR="$(

run-catch-rate.sh on GitHub

cd

"

$EVALS_DIR

/../../../.." &&

pwd

)"

EVAL_SET="$EVALS_DIR/catch-rate-eval.json"

id="$1"

config="$2"

run="$3"

: "${CLONES_DIR:?set CLONES_DIR}"

: "${OUT_DIR:?set OUT_DIR}"

[[ "$OUT_DIR" == /* ]] || { echo "OUT_DIR must be an absolute path" >&2; exit 1; }

model="${MODEL:-opus}"

[[ "$id" =~ ^[0-9]+$ ]] || { echo "eval id must be numeric" >&2; exit 1; }

[[ "$run" =~ ^[0-9]+$ ]] || { echo "run number must be numeric" >&2; exit 1; }

case "$config" in with_skill|without_skill) ;; *) echo "unknown configuration: $config" >&2; exit 1 ;; esac

case_json="$(jq -ce --argjson id "$id" '.evals[] | select(.id == $id)' "$EVAL_SET")"

repo="$(jq -r .repo <<<"$case_json")"

pr_url="$(jq -r .pr_url <<<"$case_json")"

head_sha="$(jq -r .head_sha <<<"$case_json")"

[[ "$repo" =~ ^bitwarden/[A-Za-z0-9._-]+$ ]] || { echo "unexpected repo: $repo" >&2; exit 1; }

[[ "$head_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "unexpected head_sha: $head_sha" >&2; exit 1; }

run_dir="$OUT_DIR/$id/$config/run-$run"

checkout="$run_dir/repo"

report_dir="$run_dir/report"

mkdir -p "$report_dir"

# A shared clone keeps the user's clone untouched apart from the fetched head commit.

# The remote is pointed back at GitHub so `gh pr diff <number>` resolves the PR's repo from the checkout.

clone="$CLONES_DIR/${repo#bitwarden/}"

git -C "$clone" cat-file -e "$head_sha^{commit}" 2>/dev/null || git -C "$clone" fetch -q origin "$head_sha"

if [[ ! -d "$checkout/.git" ]]; then

git clone -q --shared --no-checkout "$clone" "$checkout"

git -C "$checkout" remote set-url origin "https://github.com/$repo.git"

git -C "$checkout" checkout -q --detach "$head_sha"

# Branches and tags from the source clone reach the later fix; drop them so a run cannot read the answer.

git -C "$checkout" for-each-ref --format='delete %(refname)' | git -C "$checkout" update-ref --no-deref --stdin

fi

if [[ "$config" == with_skill ]]; then

prompt="/bitwarden-code-review:performing-multi-agent-code-review $pr_url --output-dir $report_dir"

plugin_args=(--plugin-dir "$PLUGINS_DIR/bitwarden-code-review" --plugin-dir "$PLUGINS_DIR/bitwarden-security-engineer")

else

prompt="Review pull request $pr_url for defects. The working tree is checked out at the pull request's head commit. Write the review as markdown to $report_dir/review.md. Put every finding under a \`## Findings\` heading, each in its own subsection that gives its location as \`path/to/file.ext:<line>\` or \`path/to/file.ext:<start>-<end>\`."

plugin_args=(--plugin-dir "$PLUGINS_DIR/bitwarden-security-engineer")

fi

# A corpus PR is untrusted input. dontAsk denies every call not allowed here. The deny list closes

# options that write files (--output) or run commands (--upload-pack, which git also accepts abbreviated down to --upl) under an allowed git prefix.

allowed=(

Read Grep Glob Skill Agent

"Edit(/$report_dir/**)"

"Bash(gh pr view:*)" "Bash(gh pr diff:*)"

"Bash(git diff:*)" "Bash(git status:*)" "Bash(git log:*)" "Bash(git show:*)" "Bash(git rev-parse:*)"

"Bash(git merge-base:*)" "Bash(git ls-files:*)"

"Bash(ls:*)" "Bash(cat:*)" "Bash(head:*)" "Bash(wc:*)"

)

denied=(

"Bash(git * --output*)" "Bash(git *--output=*)"

"Bash(git *--upl*)"

)

# Without this note, one denied compound command led a run to stop using Bash altogether.

harness_note="Tool permissions in this session are an allow-list, enforced per command. Allowed Bash commands: gh pr view, gh pr diff, and read-only git subcommands (diff, status, log, show, rev-parse, merge-base, ls-files), plus ls, cat, head, and wc. Run each command on its own, not chained with && or ;. A denied command does not mean Bash is unavailable."

# --setting-sources local reads .claude/settings.local.json from the checkout, which a corpus PR can commit.

[[ ! -e "$checkout/.claude/settings.local.json" && ! -L "$checkout/.claude/settings.local.json" ]] \

|| { echo "corpus checkout carries .claude/settings.local.json; refusing to load it" >&2; exit 1; }

cd "$checkout"

env -u CLAUDECODE claude -p "$prompt" \

--model "$model" \

--setting-sources local \

--permission-mode dontAsk \

--allowedTools "${allowed[@]}" \

--disallowedTools "${denied[@]}" \

--append-system-prompt "$harness_note" \

--no-session-persistence \

--output-format stream-json --verbose \

"${plugin_args[@]}" \

>"$run_dir/run.jsonl" 2>"$run_dir/stderr.log" || echo "claude exited $? for $id/$config/run-$run" >&2