Binary Checks

1// Runs the deterministic binary checks of the Instance AI eval suite on each workflow of a run.
2// Run it with tsx from the instance-ai package, because the checks are TypeScript sources:
3//
4//   pnpm --filter @n8n/instance-ai exec tsx <skill>/scripts/binary_checks.mts <run-dir>
5//
6// Writes <builder>/binary-checks.json with {source, outcomes: [{name, description, dimension, status, comment}]}.
7import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
8import { dirname, join, resolve } from 'node:path';
9import { fileURLToPath, pathToFileURL } from 'node:url';
10
11const repo = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
12const SOURCE = 'packages/@n8n/instance-ai/evaluations/binaryChecks/checks';
13
14type Check = {
15	name: string;
16	description: string;
17	kind: 'deterministic' | 'llm';
18	dimension: string;
19	run(workflow: unknown, ctx: Record<string, unknown>): unknown;
20};
21type Result = { pass: boolean; comment?: string; applicable?: boolean; errored?: boolean };
22
23const readJson = (file: string): unknown => (existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : null);
24
25const runDir = resolve(process.argv[2] ?? '.');
26const { DETERMINISTIC_CHECKS } = (await import(pathToFileURL(join(repo, SOURCE, 'index.ts')).href)) as { DETERMINISTIC_CHECKS: Check[] };
27const prompt = existsSync(join(runDir, 'prompt.txt')) ? readFileSync(join(runDir, 'prompt.txt'), 'utf8') : '';
28
29for (const entry of readdirSync(runDir, { withFileTypes: true })) {
30	const dir = join(runDir, entry.name);
31	const workflow = entry.isDirectory() ? readJson(join(dir, 'workflow.json')) : null;
32	if (!workflow) continue;
33	const outcomes = [];
34	for (const check of DETERMINISTIC_CHECKS) {
35		let status: string;
36		let comment: string | undefined;
37		try {
38			const r = (await check.run(workflow, { prompt })) as Result;
39			status = r.errored ? 'error' : r.applicable === false ? 'n_a' : r.pass ? 'pass' : 'fail';
40			comment = r.comment;
41		} catch (error) {
42			status = 'error';
43			comment = error instanceof Error ? error.message : String(error);
44		}
45		outcomes.push({ name: check.name, description: check.description, dimension: check.dimension, status, ...(comment ? { comment } : {}) });
46	}
47	writeFileSync(join(dir, 'binary-checks.json'), JSON.stringify({ source: SOURCE, outcomes }, null, 1));
48	const scored = outcomes.filter((o) => o.status === 'pass' || o.status === 'fail');
49	console.log(`${entry.name}: binary-checks.json written (${scored.filter((o) => o.status === 'pass').length}/${scored.length})`);
50}