Setting the file. One moment. scripts/inspect_eval_uv.py
scripts/inspect_eval_uv.py
Python·103 lines·3 KB
argparse
17import os
18import subprocess
19import sys
20from pathlib import Path
21from typing import Optional
22
23
24def _inspect_evals_tasks_root() -> Optional[Path]:
25 """Return the installed inspect_evals package path if available."""
26 try:
27 import inspect_evals
28
29 return Path(inspect_evals.__file__).parent
30 except Exception:
31 return None
32
33
34def _normalize_task(task: str) -> str:
35 """Allow lighteval-style `suite|task|shots` strings by keeping the task name."""
36 if "|" in task:
37 parts = task.split("|")
38 if len(parts) >= 2 and parts[1]:
39 return parts[1]
40 return task
41
42
43def main() -> None:
44 parser = argparse.ArgumentParser(description="Inspect-ai job runner")
45 parser.add_argument("--model", required=True, help="Model ID on Hugging Face Hub")
46 parser.add_argument("--task", required=True, help="inspect-ai task to execute")
47 parser.add_argument("--limit", type=int, default=None, help="Limit number of samples to evaluate")
48 parser.add_argument(
49 "--tasks-root",
50 default=None,
51 help="Optional path to inspect task files. Defaults to the installed inspect_evals package.",
52 )
53 parser.add_argument(
54 "--sandbox",
55 default="local",
56 help="Sandbox backend to use (default: local for HF jobs without Docker).",
57 )
58 args = parser.parse_args()
59
60 # Ensure downstream libraries can read the token passed as a secret
61 hf_token = os.getenv("HF_TOKEN")
62 if hf_token:
63 os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", hf_token)
64 os.environ.setdefault("HF_HUB_TOKEN", hf_token)
65
66 task = _normalize_task(args.task)
67 tasks_root = Path(args.tasks_root) if args.tasks_root else _inspect_evals_tasks_root()
68 if tasks_root and not tasks_root.exists():
69 tasks_root = None
70
71 cmd = [
72 "inspect",
73 "eval",
74 task,
75 "--model",
76 f"hf-inference-providers/{args.model}",
77 "--log-level",
78 "info",
79 # Reduce batch size to avoid OOM errors (default is 32)
80 "--max-connections",
81 "1",
82 # Set a small positive temperature (HF doesn't allow temperature=0)
83 "--temperature",
84 "0.001",
85 ]
86
87 if args.sandbox:
88 cmd.extend(["--sandbox", args.sandbox])
89
90 if args.limit:
91 cmd.extend(["--limit", str(args.limit)])
92
93 try:
94 subprocess.run(cmd, check=True, cwd=tasks_root)
95 print("Evaluation complete.")
96 except subprocess.CalledProcessError as exc:
97 location = f" (cwd={tasks_root})" if tasks_root else ""
98 print(f"Evaluation failed with exit code {exc.returncode}{location}", file=sys.stderr)
99 raise
100
101
102if __name__ == "__main__":
103 main()
104