Standard Eval Ts

1import { describe, expect } from 'vitest';
2import { evalTest } from './test-helper.js';
3
4describe('core_feature', () => {
5  // New tests MUST start as USUALLY_PASSES
6  evalTest('USUALLY_PASSES', {
7    name: 'should perform expected agent action',
8    setup: async (rig) => {
9      // For mocking offline MCP:
10      // rig.addMockMcpServer('workspace-server', 'google-workspace');
11    },
12    files: {
13      'src/app.ts': '// some code',
14    },
15    prompt: 'Task description here',
16    timeout: 60000, // 1 minute safety limit
17    assert: async (rig, result) => {
18      // 1. Audit the trajectory (Safe for standard evalTest)
19      const logs = rig.readToolLogs();
20      const hasTool = logs.some((l) => l.toolRequest.name === 'read_file');
21      expect(hasTool, 'Agent should have read the file').toBe(true);
22
23      // 2. Assert efficiency (Cost/Turn)
24      expect(logs.length).toBeLessThan(5);
25
26      // 3. Assert final output
27      expect(result).toContain('Expected Keyword');
28    },
29  });
30});