Evals

1{
2  "skill_name": "optimize-agent-prompt",
3  "evals": [
4    {
5      "id": 1,
6      "prompt": "Use Optimize Agent Prompt to create a POC that looks up a company in a state registry, improves the system prompt from run traces, and stops after it confirms convergence.",
7      "expected_output": "An isolated experiment workspace with a fixed task/schema, versioned prompts, raw Agent messages and session logs per run, a comparison report, and a winning prompt confirmed unchanged.",
8      "assertions": [
9        "The task and result schema remain fixed while system prompts change",
10        "Each prompt change is tied to one observable trace failure",
11        "The workflow retrieves both run messages and session logs when a browser session exists",
12        "The winning prompt is rerun unchanged before convergence is claimed"
13      ]
14    },
15    {
16      "id": 2,
17      "prompt": "My Browserbase Agent keeps looping after a protected site returns 403. Use the self-improving Agent API loop to make it fail fast without losing useful fallback results.",
18      "expected_output": "A bounded baseline and improved run whose prompt caps repeated retries, preserves qualified fallback completion, and compares quality, status, messages, and duration.",
19      "assertions": [
20        "The harness enforces a message or time budget and can stop a spiraling run",
21        "The teacher inspects the first consequential tool failure before editing the prompt",
22        "The evaluator does not reward a shorter failed or unsafe run as an improvement"
23      ]
24    },
25    {
26      "id": 3,
27      "prompt": "Build an Agent API prompt optimizer for a workflow that could submit a real payment. Explore and improve navigation, but never let evaluation runs submit the payment.",
28      "expected_output": "A task and baseline prompt with an explicit irreversible-action boundary, structured evidence of the pre-submit state, and iterations that preserve the boundary while optimizing the prompt.",
29      "assertions": [
30        "The experiment encodes the irreversible payment action as forbidden before any run",
31        "Factuality or safety warnings penalize evidence of a forbidden submission",
32        "No iteration broadens authority merely to complete the task"
33      ]
34    }
35  ]
36}