1#!/usr/bin/env python32"""Check what the final message of each builder says against the trace, the workflow and the runs.34 claims.py <run-dir>56Writes <builder>/claims.json with {message, checks: [{id, ok, text, details}]}. No LLM is involved.7A check is about one kind of claim: the workflow id, the publish state, validation, testing, the8values in `code spans`, and if a claim that the workflow works agrees with the service-call runs.9A message that does not make a kind of claim gets no check for it.10"""1112import json13import re14import sys15
69 return [f'Message: "{c}"' for c in clauses_[:3]]
70
71
72def id_check(message, meta, workflow):
73 ids = {a or b for a, b in WORKFLOW_ID.findall(message or '')}
74 if not ids:
75 return None
76 known = set(meta.get('workflowIds') or []) | {meta.get('workflowId'), (workflow or {}).get('id')}
77 wrong = sorted(ids - known)
78 return check('M1', not wrong, 'The workflow id in the message is the workflow that the run created',
79 [f'{i} is not a workflow of this run' for i in wrong] or [f'{", ".join(sorted(ids))} matches.'])
80
81
82def state_check(message, workflow):
83 draft, live = DRAFT.findall(message or ''), LIVE.findall(message or '')
84 if workflow is None or not (draft or live):
85 return None
86 active = bool(workflow.get('active'))
87 says = 'active' if live and not draft else 'a draft'
88 ok = active == (says == 'active')
89 return check('M2', ok, f'The message says the workflow is {says}', [f'workflow.json active is {str(active).lower()}.'])
90
91
92def validation_check(message, events):
93 pos, _ = claim(message, VALIDATED, PASSED)
94 if not pos:
95 return None
96 calls = [e for e in events if (e['kind'] == 'tool' and VALIDATE_TOOL.search(e.get('name') or ''))
97 or (e['kind'] == 'code' and VALIDATE_TOOL.search(e.get('code') or ''))]
98 if not calls:
99 return check('M3', False, 'The message says the workflow passed validation', quote(pos) + ['No validation tool call in the trace.'])
100 last = calls[-1]
101 failed = last.get('error') or INVALID.search(str(last.get('result') or ''))
102 return check('M3', not failed, 'The message says the workflow passed validation', quote(pos) + [
103 f'{len(calls)} validation call(s). The last one {"failed" if failed else "passed"}: {last.get("name")}.'])
104
105
106def test_check(message, events):
107 pos, neg = claim(message, TESTED)
108 runs = [e for e in events if e['kind'] == 'tool' and TEST_TOOL.search(e.get('name') or '')
109 and not VALIDATE_TOOL.search(e.get('name') or '') and not SHELL_TOOL.search(e.get('name') or '')]
110 scripts = [e for e in events if e['kind'] == 'code' and SCRIPT_PASSED.search(str(e.get('result') or ''))]
111 if pos:
112 ok_runs = [e for e in runs if not e.get('error') and not RUN_FAILED.search(tool_text({'result': e.get('result')}))]
113 if ok_runs:
114 names = sorted({e['name'] for e in ok_runs})
115 return check('M4', True, 'The message says the workflow was tested', quote(pos) + [f'{len(ok_runs)} successful test call(s): {", ".join(names)}.'])
116 if scripts:
117 return check('M4', True, 'The message says the workflow was tested', quote(pos) + [
118 f'{len(scripts)} script(s) in the trace report passed tests. These ran outside n8n, not as a workflow execution.'])
119 return check('M4', False, 'The message says the workflow was tested', quote(pos) + ['No test or execution call in the trace.'])
120 if neg and runs:
121 return check('M4', False, 'The message says the workflow was not run', quote(neg) + [f'The trace has {len(runs)} test call(s).'])
122 if neg:
123 return check('M4', True, 'The message says the workflow was not run', quote(neg) + ['No test or execution call in the trace.'])
134 missing = [s for s in spans if loose(s) not in hay]
135 return check('M5', not missing, f'Values in code spans are in the workflow, the prompt or a tool result ({len(spans) - len(missing)} of {len(spans)})',
136 [f'`{s}` is not in the workflow, the prompt or any tool call' for s in missing])