Make Workspace

1#!/usr/bin/env bash
2# Create the eval workspace directory tree for a run.
3#
4# Usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]
5#   workspace   Run root, e.g. /private/tmp/expo-skill-eval-<skill>
6#   iteration   Iteration folder name, e.g. "iteration-1"
7#   num-evals   Number of eval cases; creates eval-0 .. eval-(N-1)
8#   configs     Space- or comma-separated config names
9#               (default: "with_skill without_skill")
10#
11# Creates <workspace>/trigger-evals/scratch and, per eval x config,
12# <workspace>/<iteration>/eval-<i>/<config>/outputs. Run this once at the
13# start of a run instead of ad-hoc `mkdir` so the call is covered by the
14# skill's `Bash(bash *expo-skill-eval/scripts/*)` rule (the mkdir calls
15# inside run as children of this script and need no permission of their own).
16set -euo pipefail
17
18WORKSPACE="${1:?usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]}"
19ITERATION="${2:?usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]}"
20NUM_EVALS="${3:?usage: make-workspace.sh <workspace> <iteration> <num-evals> [configs]}"
21CONFIGS="${4:-with_skill without_skill}"
22CONFIGS="${CONFIGS//,/ }"
23
24mkdir -p "$WORKSPACE/trigger-evals/scratch"
25for ((i = 0; i < NUM_EVALS; i++)); do
26  for cfg in $CONFIGS; do
27    mkdir -p "$WORKSPACE/$ITERATION/eval-$i/$cfg/outputs"
28  done
29done
30
31echo "workspace ready: $WORKSPACE/$ITERATION ($NUM_EVALS evals x [$CONFIGS])"