Setting the file. One moment.
Summarize Runs · N8n:testmax N8n AI · n8n-io/n8n · Skills Docs
Back to the top of the page
Type
Python scripts/ summarize_runs.py
Python · 215 lines · 9 KB
16 import json
17 import random
18 import re
19 import sys
20 from pathlib import Path
21
22 sys.path.insert( 0 , str (Path( __file__ ).resolve().parent))
23 from build_report import load_case, parse_criteria # noqa: E402
24
25 NODE_SETTINGS = ( 'disabled' , 'onError' , 'retryOnFail' , 'maxTries' , 'waitBetweenTries' , 'executeOnce' ,
26 'alwaysOutputData' , 'notesInFlow' , 'notes' )
27 BUILDER_NAMES = re.compile(
28 r ' \b( Instance AI | n8n Assistant | Claude Code | Claude | Anthropic | Codex | OpenAI | ChatGPT | GPT- [\w .- ] +| Sonnet | Opus | Haiku )\b ' ,
29 re. IGNORECASE )
30 MARK = { True : '✓' , False : '✗' , None : '?' }
31 REPO = Path( __file__ ).resolve().parents[ 4 ]
32 LLM_CHECKS_DIR = REPO / 'packages/@n8n/instance-ai/evaluations/binaryChecks/checks'
33 # The LLM binary checks of the eval suite that the judges answer as quality checks.
34 QUALITY_CHECKS = ( 'valid-data-flow' , 'correct-node-operations' , 'handles-multiple-items' , 'descriptive-node-names' ,
35 'response-describes-changes-accurately' )
36
37
38 def quality_checks ():
39 """[(name, description, rules)] read from the check sources, so the rules stay those of the eval suite."""
40 out = []
41 for file in QUALITY_CHECKS :
42 path = LLM_CHECKS_DIR / f ' { file } .ts'
43 if not path.exists():
44 continue
45 text = path.read_text()
46 name = re.search( r "name: \s * ' ([ ^' ] + ) '" , text)
47 desc = re.search( r "description: \s * ([ ' \" ])(. *? ) \1 ," , text, re.S)
48 rules = re.search( r 'systemPrompt: \s * ` ((?: \\ ` | [ ^` ]) * ) `' , text)
49 if name and rules:
50 out.append((name.group( 1 ), desc.group( 2 ) if desc else '' , rules.group( 1 ).replace( ' \\ `' , '`' ).strip()))
51 return out
52
53
54 def quality_text ():
55 checks = quality_checks()
56 if not checks:
57 return ''
58 lines = [ 'QUALITY CHECKS (from the Instance AI eval suite). For each builder, answer each check once in "quality",' ,
59 'with the check name as the id. The final message is the narration of the agent. Each build starts' ,
60 'from an empty workflow, so the START workflow is empty.' ]
61 for name, desc, rules in checks:
62 lines += [ f '--- [ { name } ] { desc } ' , rules]
63 return ' \n ' .join(lines) + ' \n '
64
65
66 def session_final_message (sub, meta):
67 """Final agent message from the session file of an interactive CLI run."""
68 if not meta.get( 'sessionFile' ):
69 return None
70 text = None
71 for line in (sub / meta[ 'sessionFile' ]).read_text().splitlines():
72 row = json.loads(line)
73 payload = row.get( 'payload' ) or {}
74 if row.get( 'type' ) == 'event_msg' and payload.get( 'type' ) == 'task_complete' :
75 text = payload.get( 'last_agent_message' ) or text
76 if row.get( 'type' ) == 'assistant' :
77 blocks = (row.get( 'message' ) or {}).get( 'content' ) or []
78 texts = [b[ 'text' ] for b in blocks if isinstance (b, dict ) and b.get( 'type' ) == 'text' and b.get( 'text' , '' ).strip()]
79 text = ' \n ' .join(texts) or text
80 return text
81
82
83 def final_message (sub, meta):
84 kind = meta.get( 'kind' )
85 if kind == 'instance-ai' :
86 trace = json.loads((sub / 'trace.json' ).read_text())
87 rich = trace[ 'rich' ].get( 'data' , trace[ 'rich' ])
88 assistant = [m for m in rich[ 'messages' ] if m.get( 'role' ) == 'assistant' ]
89 return (assistant[ - 1 ].get( 'agentTree' ) or {}).get( 'textContent' ) or assistant[ - 1 ].get( 'content' ) if assistant else None
90 if not (sub / 'stream.jsonl' ).exists():
91 return session_final_message(sub, meta)
92 text = None
93 for line in (sub / 'stream.jsonl' ).read_text().splitlines():
94 event = json.loads(line)
95 if event.get( 'type' ) == 'result' :
96 text = event.get( 'result' )
97 item = event.get( 'item' ) or {}
98 if event.get( 'type' ) == 'item.completed' and item.get( 'type' ) == 'agent_message' :
99 text = item.get( 'text' )
100 return text
101
102
103 def connections (workflow):
104 lines = []
105 for source, outputs in (workflow.get( 'connections' ) or {}).items():
106 for kind, branches in outputs.items():
107 for index, targets in enumerate (branches or []):
108 for target in targets or []:
109 branch = f '[ { index } ]' if len (branches) > 1 else ''
110 # Merge and other multi-input nodes need the input index to show which side an edge feeds.
111 target_index = target.get( 'index' ) or 0
112 inlet = f '[in { target_index } ]' if target_index else ''
113 via = '' if kind == 'main' else f ' ( { kind } )'
114 lines.append( f ' { source }{ branch } -> { target[ "node" ] }{ inlet }{ via } ' )
115 return lines
116
117
118 def criteria_text (expected):
119 """The expected outcome with the id of each criterion, as the judge must cite it."""
120 groups = parse_criteria(expected or '' )
121 if not groups:
122 return (expected or '(none)' ).strip()
123 lines = []
124 for heading, items in groups:
125 if heading:
126 lines.append( f ' { heading } :' )
127 lines.extend( f ' [ { num } ] { text } ' for num, text in items)
128 return ' \n ' .join(lines)
129
130
131 def auto_check_lines (sub):
132 path = sub / 'auto-checks.json'
133 if not path.exists():
134 return [ ' (none: derive_checks.mjs did not run)' ]
135 lines = []
136 for group in json.loads(path.read_text()).get( 'groups' , []):
137 score = group.get( 'score' )
138 suffix = f ' · score { score[ "passed" ] } / { score[ "total" ] } ' if score else ''
139 lines.append( f ' { group[ "title" ] }{ suffix } — { group.get( "note" , "" ) } ' )
140 for c in group.get( 'checks' , []):
141 mark = 'i' if c.get( 'level' ) == 'info' else MARK .get(c.get( 'ok' ), '?' )
142 lines.append( f ' [ { mark } ] { c[ "id" ] } { c[ "text" ] } ' )
143 lines.extend( f ' · { d } ' for d in c.get( 'details' ) or [])
144 return lines
145
146
147 def run_section (sub, meta, label, blind = False ):
148 out = [ f '===== { label } workflow= { meta.get( "workflowId" ) } =====' ]
149 try :
150 message = (final_message(sub, meta) or '(none)' ).strip()
151 out.append( 'FINAL MESSAGE: \n ' + (scrub(message, meta) if blind else message))
152 except Exception as e:
153 out.append( f 'FINAL MESSAGE: unreadable ( { e } )' )
154 wf_file = sub / 'workflow.json'
155 if not wf_file.exists():
156 out.append( 'WORKFLOW: none \n ' )
157 return ' \n ' .join(out)
158 workflow = json.loads(wf_file.read_text())
159 out.append( f 'WORKFLOW: { workflow.get( "name" ) } nodes= { len (workflow.get( "nodes" , [])) } ' )
160 out.append( f 'SETTINGS: { json.dumps(workflow.get( "settings" ) or {}, ensure_ascii = False ) } ' )
161 for node in workflow.get( 'nodes' , []):
162 creds = ', ' .join((node.get( 'credentials' ) or {}).keys())
163 out.append( f '- { node[ "name" ] } | { node[ "type" ] } v { node.get( "typeVersion" ) } ' + ( f ' | creds: { creds } ' if creds else '' ))
164 settings = {k: node[k] for k in NODE_SETTINGS if k in node}
165 if settings:
166 out.append( f ' (settings) { json.dumps(settings, ensure_ascii = False ) } ' )
167 for key, value in (node.get( 'parameters' ) or {}).items():
168 text = value if isinstance (value, str ) else json.dumps(value, ensure_ascii = False )
169 out.append( f ' { key } : { text } ' )
170 out.append( 'CONNECTIONS:' )
171 out.append( ' \n ' .join(connections(workflow)) or ' (none)' )
172 out.append( 'AUTO-CHECK FACTS (deterministic, from the node definitions, the n8n editor, pinned-data runs, '
173 'in-process runs with recorded API requests, the eval binary checks and the final-message claims):' )
174 out.extend(auto_check_lines(sub))
175 return ' \n ' .join(out) + ' \n '
176
177
178 def scrub (text, meta):
179 """Remove builder and model names from a final message. The workflow itself stays as built."""
180 for name in filter ( None , [meta.get( 'model' ), meta.get( 'kind' )]):
181 text = re.sub(re.escape(name), 'the builder' , text, flags = re. IGNORECASE )
182 return BUILDER_NAMES .sub( 'the builder' , text)
183
184
185 def main ():
186 ap = argparse.ArgumentParser( description = 'Print the evidence of each run for the judge step.' )
187 ap.add_argument( 'run_dir' , nargs = '?' , default = '.' )
188 ap.add_argument( '--blind' , action = 'store_true' , help = 'Write a summary with the builders called A, B and C.' )
189 ap.add_argument( '--seed' , type = int , help = 'Seed of the blind order, for a reproducible test.' )
190 args = ap.parse_args()
191 run_dir = Path(args.run_dir)
192 case = load_case(run_dir)
193 runs = [(f.parent, json.loads(f.read_text())) for f in sorted (run_dir.glob( '*/run.json' )) if f.parent.name != 'report' ]
194 header = ( 'EXPECTED OUTCOME CRITERIA (cite each criterion by its id in brackets): \n '
195 f ' { criteria_text(case.get( "expected" )) }\n\n{ quality_text() } ' )
196
197 if not args.blind:
198 print (header)
199 for sub, meta in runs:
200 print (run_section(sub, meta, f ' { sub.name } ( { meta.get( "kind" ) } )' ))
201 return
202
203 order = runs[:]
204 random.Random(args.seed).shuffle(order)
205 letters = [ chr ( ord ( 'A' ) + i) for i in range ( len (order))]
206 sections = [run_section(sub, meta, f 'Builder { letter } ' , blind = True ) for letter, (sub, meta) in zip (letters, order)]
207 judge_dir = run_dir / 'judge'
208 judge_dir.mkdir( exist_ok = True )
209 (judge_dir / 'summary-blind.txt' ).write_text(header + ' \n ' + ' \n ' .join(sections))
210 (judge_dir / 'blind-map.json' ).write_text(json.dumps({letter: sub.name for letter, (sub, _) in zip (letters, order)}, indent = 2 ))
211 print (judge_dir / 'summary-blind.txt' )
212
213
214 if __name__ == '__main__' :
215 main()