Setting the file. One moment.
Build Report · N8n:testmax N8n AI · n8n-io/n8n · Skills Docs
Back to the top of the page — line 201
This file
Number 40.2
Position 2 of 24
Type Python
Size 113 KB
Lines 2,309 scripts/ build_report.py
Python · 2,309 lines · 113 KB
16 <name>/run.json and the files that the runner scripts wrote for each builder
17 verdicts.json optional: {"<name>": {"verdict": "pass|partial|fail", "summary": "...",
18 "checks": [{"id": 3, "ok": true|false|null, "text": "..."}]}}
19 id is the number of the expected outcome item. It is optional.
20
21 Output: <run-dir>/report/index.html with the canvas images copied next to it.
22 """
23
24 import argparse
25 import html
26 import json
27 import os
28 import re
29 import shutil
30 import sys
31 import time
32 import urllib.request
33 from collections import Counter
34 from datetime import datetime
35 from pathlib import Path
36
37 ASSET_CACHE = Path.home() / '.config' / 'n8n-builder-compare' / 'assets'
38 PLAYER_VERSION = '3.17.0'
39 PLAYER_FILES = ( 'asciinema-player.min.js' , 'asciinema-player.css' )
40 MAX_CHARS = 15000
41 TITLES = { 'instance-ai' : 'Instance AI' , 'claude' : 'Claude Code + n8n MCP' , 'codex' : 'Codex + n8n MCP' }
42
43
44 def ts (value):
45 value = value.replace( 'Z' , '+00:00' )
46 if '+' not in value[ 10 :] and '-' not in value[ 19 :]:
47 value += '+00:00'
48 return datetime.fromisoformat(value).timestamp()
49
50
51 def esc (value):
52 return html.escape(value if isinstance (value, str ) else json.dumps(value, indent = 2 , ensure_ascii = False ))
53
54
55 def pretty (value):
56 if isinstance (value, str ):
57 try :
58 value = json.loads(value)
59 except ValueError :
60 pass
61 text = value if isinstance (value, str ) else json.dumps(value, indent = 2 , ensure_ascii = False )
62 if len (text) > MAX_CHARS :
63 text = text[: MAX_CHARS ] + f ' \n\n … truncated ( { len (text) :,} chars in total)'
64 return text
65
66
67 def content_text (content):
68 if isinstance (content, dict ) and 'content' in content:
69 content = content[ 'content' ]
70 if isinstance (content, list ):
71 return ' \n ' .join(c.get( 'text' , '' ) if isinstance (c, dict ) else str (c) for c in content)
72 return content if isinstance (content, str ) else json.dumps(content)
73
74
75 def kilo (n):
76 return f ' { n / 1_000_000 :.2f} M' if n >= 1_000_000 else f ' { n / 1000 :.1f} k' if n >= 1000 else str (n)
77
78
79 def price_calls (calls, model, prices):
80 """Calculate the cost of each call from the case.json price of the model."""
81 p = (prices or {}).get(model)
82 if not p:
83 return False
84 for c in calls:
85 fresh = max (c[ 'prompt' ] - c[ 'cache_read' ] - c[ 'cache_write' ], 0 )
86 c[ 'cost' ] = (
87 fresh * p.get( 'input' , 0 ) + c[ 'cache_read' ] * p.get( 'cached' , 0 )
88 + c[ 'cache_write' ] * p.get( 'cacheWrite' , p.get( 'input' , 0 )) + c[ 'out' ] * p.get( 'output' , 0 )
89 ) / 1_000_000
90 return True
91
92
93 def png_size (path):
94 """Width and height in pixels, read from the PNG header."""
95 header = Path(path).read_bytes()[ 16 : 24 ]
96 return int .from_bytes(header[: 4 ], 'big' ), int .from_bytes(header[ 4 :], 'big' )
97
98
99 def read_jsonl (path):
100 lines = Path(path).read_text().splitlines()
101 rows = []
102 for i, line in enumerate (lines):
103 if not line.strip():
104 continue
105 try :
106 rows.append(json.loads(line))
107 except json.JSONDecodeError:
108 # A running builder can be in the middle of writing the last line.
109 if i == len (lines) - 1 :
110 break
111 raise
112 return rows
113
114
115 def run_seconds (meta):
116 # A CLI run also spends time on container start-up, the prompt paste wait and shutdown.
117 if 'turnStartedAt' in meta or 'turnFinishedAt' in meta:
118 if meta.get( 'turnStartedAt' ) and meta.get( 'turnFinishedAt' ):
119 return ts(meta[ 'turnFinishedAt' ]) - ts(meta[ 'turnStartedAt' ])
120 return None
121 return ts(meta[ 'finishedAt' ]) - ts(meta[ 'startedAt' ]) if meta.get( 'finishedAt' ) and meta.get( 'startedAt' ) else None
122
123
124 def load_claude (run_dir, meta, prices):
125 rows = read_jsonl(run_dir / meta[ 'sessionFile' ])
126 # Interactive runs have no event stream, so there is no result event with a cost estimate.
127 stream = read_jsonl(run_dir / 'stream.jsonl' ) if (run_dir / 'stream.jsonl' ).exists() else []
128 final = next ((d for d in stream if d.get( 'type' ) == 'result' ), {})
129 start = ts( next (r[ 'timestamp' ] for r in rows if r.get( 'timestamp' )))
130 results = {}
131 for r in rows:
132 content = r[ 'message' ].get( 'content' ) if r.get( 'type' ) == 'user' else None
133 for c in content if isinstance (content, list ) else []:
134 if c.get( 'type' ) == 'tool_result' :
135 results[c[ 'tool_use_id' ]] = (r[ 'timestamp' ], c)
136
137 # One API response is split over several rows that repeat its usage. The last copy is complete.
138 calls, events, model = {}, [], None
139 for r in rows:
140 if r.get( 'type' ) != 'assistant' :
141 continue
142 msg = r[ 'message' ]
143 model = msg.get( 'model' ) or model
144 t = ts(r[ 'timestamp' ]) - start
145 usage = msg.get( 'usage' ) or {}
146 read, write, inp = (usage.get( 'cache_read_input_tokens' ) or 0 , usage.get( 'cache_creation_input_tokens' ) or 0 ,
147 usage.get( 'input_tokens' ) or 0 )
148 # Anthropic reports uncached input only.
149 prompt = inp + read + write
150 calls.setdefault(msg[ 'id' ], { 't' : t, 'id' : msg[ 'id' ]}).update(
151 prompt = prompt, cache_read = read, cache_write = write, out = usage.get( 'output_tokens' ) or 0 , cost = None )
152 for c in msg.get( 'content' , []):
153 kind = c.get( 'type' )
154 if kind == 'thinking' and c.get( 'thinking' , '' ).strip():
155 events.append({ 'kind' : 'reasoning' , 't' : t, 'text' : c[ 'thinking' ], 'call' : msg[ 'id' ]})
156 elif kind == 'text' and c.get( 'text' , '' ).strip():
157 events.append({ 'kind' : 'text' , 't' : t, 'text' : c[ 'text' ], 'call' : msg[ 'id' ]})
158 elif kind == 'tool_use' :
159 end, res = results.get(c[ 'id' ], ( None , {}))
160 events.append({
161 'kind' : 'tool' , 't' : t, 'call' : msg[ 'id' ], 'name' : split_tool_name(c[ 'name' ])[ 1 ], 'fullName' : c[ 'name' ],
162 'duration' : (ts(end) - start - t) if end else None ,
163 'args' : c.get( 'input' ), 'result' : content_text(res.get( 'content' )),
164 'error' : bool (res.get( 'is_error' )),
165 })
166 calls = list (calls.values())
167 if price_calls(calls, model, prices):
168 cost, source = f '$ { sum (c[ "cost" ] for c in calls) :.2f} ' , 'prices'
169 elif final.get( 'total_cost_usd' ) is not None :
170 cost, source = f '$ { final[ "total_cost_usd" ] :.2f} ' , 'claude-code'
171 else :
172 cost, source = 'not reported' , None
173 duration = final[ 'duration_ms' ] / 1000 if final.get( 'duration_ms' ) else run_seconds(meta)
174 return { 'events' : events, 'calls' : calls, 'model' : model, 'context' : claude_context(rows),
175 'duration' : f ' { duration :.0f} s' if duration else '-' , 'cost' : cost, 'costSource' : source}
176
177
178 def claude_context (rows):
179 """The system prompt and tools of the first API request, from the request shape in the session file."""
180 shape = next ((r[ 'shape' ] for r in rows if r.get( 'type' ) == 'api-request-shape' ), None )
181 if not shape:
182 return None
183 system = shape.get( 'system' )
184 text = system if isinstance (system, str ) else ' \n\n ' .join(b.get( 'text' , '' ) for b in system or [])
185 tools = [{ 'name' : t.get( 'name' ), 'description' : t.get( 'description' ) or '' , 'schema' : t.get( 'input_schema' ),
186 'type' : t.get( 'type' ), 'deferred' : bool (t.get( 'defer_loading' ))} for t in shape.get( 'tools' ) or []]
187 # With tool search on, the model gets only the names of deferred tools and loads a schema with ToolSearch.
188 deferred = {}
189 for r in rows:
190 delta = r.get( 'attachment' ) if isinstance (r.get( 'attachment' ), dict ) else {}
191 if delta.get( 'type' ) == 'deferred_tools_delta' :
192 deferred.update( dict .fromkeys(delta.get( 'addedNames' ) or []))
193 for name in delta.get( 'removedNames' ) or []:
194 deferred.pop(name, None )
195 known = {t[ 'name' ] for t in tools}
196 tools += [{ 'name' : name, 'description' : '' , 'schema' : None , 'type' : None , 'deferred' : True }
197 for name in deferred if name not in known]
198 return { 'messages' : [{ 'label' : 'System prompt' , 'text' : text}] if text else [], 'tools' : tools}
199
200
201 def mcp_name (item):
202 return f 'mcp__ { item[ "server" ] } __ { item.get( "tool" ) } ' if item.get( 'server' ) else None
203
204
205 def codex_session_events (rows):
206 """Trace events from the item_completed rows of a Codex session file."""
207 first = next ((ts(r[ 'timestamp' ]) for r in rows if r.get( 'timestamp' )), 0 )
208 events = []
209 # Codex calls MCP tools from scripts that it runs with its exec tool. The scripts are not n8n tool calls.
210 outputs = {(r.get( 'payload' ) or {}).get( 'call_id' ): (r.get( 'payload' ) or {}).get( 'output' ) for r in rows
211 if (r.get( 'payload' ) or {}).get( 'type' ) == 'custom_tool_call_output' }
212 for r in rows:
213 payload = r.get( 'payload' ) or {}
214 if r.get( 'type' ) == 'response_item' and payload.get( 'type' ) == 'custom_tool_call' :
215 events.append({ 'kind' : 'code' , 't' : ts(r[ 'timestamp' ]) - first, 'name' : payload.get( 'name' ),
216 'code' : payload.get( 'input' ), 'result' : content_text(outputs.get(payload.get( 'call_id' )))})
217 continue
218 if r.get( 'type' ) != 'event_msg' or payload.get( 'type' ) != 'item_completed' :
219 continue
220 item = payload.get( 'item' ) or {}
221 begin = (payload.get( 'started_at_ms' ) or 0 ) / 1000 - first if payload.get( 'started_at_ms' ) else ts(r[ 'timestamp' ]) - first
222 end = (payload.get( 'completed_at_ms' ) or 0 ) / 1000 - first if payload.get( 'completed_at_ms' ) else begin
223 kind = item.get( 'type' )
224 if kind in ( 'AgentMessage' , 'Reasoning' ):
225 text = ' \n ' .join(c.get( 'text' , '' ) for c in item.get( 'content' ) or item.get( 'summary_text' ) or []
226 if isinstance (c, dict )).strip()
227 if text:
228 events.append({ 'kind' : 'text' if kind == 'AgentMessage' else 'reasoning' , 't' : begin, 'text' : text})
229 elif kind == 'McpToolCall' :
230 result = item.get( 'result' )
231 events.append({
232 'kind' : 'tool' , 't' : begin, 'name' : item.get( 'tool' ), 'fullName' : mcp_name(item),
233 'duration' : end - begin, 'args' : item.get( 'arguments' ),
234 'result' : content_text(result) if result else item.get( 'error' ),
235 'error' : bool (item.get( 'error' )) or item.get( 'status' ) == 'failed'
236 or ( isinstance (result, dict ) and bool (result.get( 'isError' ) or result.get( 'is_error' ))),
237 })
238 return events
239
240
241 def load_codex (run_dir, meta, prices):
242 session = run_dir / meta[ 'sessionFile' ] if meta.get( 'sessionFile' ) else None
243 rows = read_jsonl(session) if session and session.exists() else []
244 tools = codex_request_tools(run_dir / (meta.get( 'requests' ) or 'requests.jsonl' ), meta.get( 'sessionId' ))
245 if not (run_dir / 'stream.jsonl' ).exists():
246 return codex_result(codex_session_events(rows), rows, meta, prices, run_seconds(meta), tools)
247 stream = read_jsonl(run_dir / 'stream.jsonl' )
248 times_file = run_dir / 'stream.times'
249 times = [ float (x) for x in times_file.read_text().split()] if times_file.exists() else [ 0.0 ] * len (stream)
250 start = times[ 0 ] if times else 0.0
251 started, events = {}, []
252 for event, at in zip (stream, times):
253 item = event.get( 'item' ) or {}
254 t = at - start
255 if event.get( 'type' ) == 'item.started' :
256 started[item.get( 'id' )] = t
257 continue
258 if event.get( 'type' ) != 'item.completed' :
259 continue
260 kind = item.get( 'type' )
261 if kind == 'agent_message' and (item.get( 'text' ) or '' ).strip():
262 events.append({ 'kind' : 'text' , 't' : t, 'text' : item[ 'text' ]})
263 elif kind == 'reasoning' and (item.get( 'text' ) or '' ).strip():
264 events.append({ 'kind' : 'reasoning' , 't' : t, 'text' : item[ 'text' ]})
265 elif kind == 'mcp_tool_call' :
266 begin = started.get(item.get( 'id' ), t)
267 events.append({
268 'kind' : 'tool' , 't' : begin, 'name' : item.get( 'tool' ), 'fullName' : mcp_name(item),
269 'duration' : t - begin, 'args' : item.get( 'arguments' ),
270 'result' : content_text(item.get( 'result' )) if item.get( 'result' ) else item.get( 'error' ),
271 'error' : bool (item.get( 'error' )) or item.get( 'status' ) == 'failed' ,
272 })
273 elif kind == 'command_execution' :
274 begin = started.get(item.get( 'id' ), t)
275 events.append({
276 'kind' : 'tool' , 't' : begin, 'name' : 'shell' , 'duration' : t - begin,
277 'args' : item.get( 'command' ), 'result' : item.get( 'aggregated_output' ),
278 'error' : item.get( 'exit_code' ) not in ( 0 , None ),
279 })
280
281 return codex_result(events, rows, meta, prices, (times[ - 1 ] - start) if times else None , tools)
282
283
284 def codex_result (events, rows, meta, prices, duration, tools = None ):
285 calls, model = [], meta.get( 'model' )
286 first = next ((ts(r[ 'timestamp' ]) for r in rows if r.get( 'timestamp' )), 0 )
287 for r in rows:
288 payload = r.get( 'payload' ) or {}
289 if r.get( 'type' ) == 'turn_context' :
290 model = payload.get( 'model' ) or model
291 if r.get( 'type' ) == 'token_usage_record' :
292 u = payload.get( 'usage' ) or {}
293 calls.append({ 't' : ts(r[ 'timestamp' ]) - first, 'prompt' : u.get( 'input_tokens' ) or 0 ,
294 'cache_read' : u.get( 'cached_input_tokens' ) or 0 ,
295 'cache_write' : u.get( 'cache_write_input_tokens' ) or 0 ,
296 'out' : u.get( 'output_tokens' ) or 0 , 'cost' : None })
297 priced = price_calls(calls, model, prices)
298 return { 'events' : events, 'calls' : calls, 'model' : model, 'context' : codex_context(rows, tools),
299 'duration' : f ' { duration :.0f} s' if duration else '-' ,
300 'cost' : f '$ { sum (c[ "cost" ] for c in calls) :.2f} ' if priced else 'not reported' ,
301 'costSource' : 'prices' if priced else None }
302
303
304 NESTED_TOOL = re.compile( r ' ^ ### ` ([ ^` ] + ) ` [ \t ] * $ ' , re.M)
305 TS_BLOCK = re.compile( r '```ts \n (. *? ) ```' , re.S)
306
307
308 def split_nested_tools (text):
309 """In code mode, the exec description lists the tools that its scripts can call, one ### section
310 each with a TypeScript declaration. Returns the description before those sections and the tools."""
311 marks = list ( NESTED_TOOL .finditer(text))
312 nested = []
313 for i, m in enumerate (marks):
314 body = text[m.end():marks[i + 1 ].start() if i + 1 < len (marks) else len (text)]
315 blocks = TS_BLOCK .findall(body)
316 if blocks:
317 nested.append((m.group( 1 ), body.split( 'exec tool declaration:' )[ 0 ].strip(), blocks[ - 1 ].strip()))
318 if not nested:
319 return text, []
320 return text[:marks[ 0 ].start()].rstrip(), nested
321
322
323 def codex_tool (t, name):
324 tool = { 'name' : name, 'description' : t.get( 'description' ) or '' , 'schema' : t.get( 'parameters' ),
325 'type' : t.get( 'type' )}
326 if t.get( 'type' ) != 'custom' :
327 return [tool]
328 fmt = t.get( 'format' ) or {}
329 if fmt.get( 'definition' ):
330 tool[ 'code' ] = { 'label' : f 'Input grammar ( { fmt.get( "syntax" ) or fmt.get( "type" ) } )' , 'text' : fmt[ 'definition' ]}
331 tool[ 'description' ], nested = split_nested_tools(tool[ 'description' ])
332 return [tool] + [{ 'name' : n, 'description' : d, 'schema' : None , 'type' : 'function' , 'group' : f 'System · in { name } ' ,
333 'code' : { 'label' : f 'Declaration in { name } ' , 'text' : decl}} for n, d, decl in nested]
334
335
336 def codex_request_tools (path, session_id = None ):
337 """The tools of the first model request of the session thread that the request recorder saved,
338 as one flat list. Side tasks, such as a title request, run on other threads with other tools."""
339 if not path.exists():
340 return None
341 rows = [r for r in read_jsonl(path) if r.get( 'tools' )]
342 own = [r for r in rows if session_id and (r.get( 'client_metadata' ) or {}).get( 'thread_id' )
343 and session_id.endswith(r[ 'client_metadata' ][ 'thread_id' ])]
344 raw = (own or rows or [{}])[ 0 ].get( 'tools' )
345 tools = []
346 for t in raw or []:
347 if t.get( 'type' ) != 'namespace' :
348 tools += codex_tool(t, t.get( 'name' ) or t.get( 'type' ))
349 continue
350 space = t.get( 'name' ) or ''
351 for sub in t.get( 'tools' ) or []:
352 name = sub.get( 'name' ) or ''
353 tools += codex_tool(sub, f ' { space.rstrip( "_" ) } __ { name } ' if space.startswith( 'mcp__' ) else name)
354 return tools or None
355
356
357 def codex_context (rows, tools = None ):
358 """The base instructions and the messages that Codex adds before the prompt, from the session file,
359 and the tools from the request recording."""
360 meta = next ((r.get( 'payload' ) or {} for r in rows if r.get( 'type' ) == 'session_meta' ), {})
361 text = (meta.get( 'base_instructions' ) or {}).get( 'text' )
362 messages = [{ 'label' : 'System prompt' , 'text' : text}] if text else []
363 for r in rows:
364 payload = r.get( 'payload' ) or {}
365 if r.get( 'type' ) != 'response_item' or payload.get( 'type' ) != 'message' :
366 continue
367 body = ' \n ' .join(c.get( 'text' , '' ) for c in payload.get( 'content' ) or [] if isinstance (c, dict )).strip()
368 # Codex wraps its own context in tags. The first untagged user message is the prompt.
369 if payload.get( 'role' ) == 'user' and not body.startswith( '<' ):
370 break
371 if payload.get( 'role' ) in ( 'developer' , 'user' ) and body:
372 messages.append({ 'label' : 'Developer message' if payload[ 'role' ] == 'developer' else 'Context' , 'text' : body})
373 if not messages and not tools:
374 return None
375 if not tools:
376 return { 'messages' : messages, 'tools' : None ,
377 'toolsNote' : 'Codex does not save its tool definitions, and this run has no request recording.' }
378 notes = {}
379 if any (t.get( 'group' ) for t in tools) and not any (split_tool_name(t[ 'name' ])[ 0 ] for t in tools):
380 notes[ 'MCP' ] = 'Not in the prompt. Codex scripts look up MCP tools in ALL_TOOLS at run time.'
381 return { 'messages' : messages, 'tools' : tools, 'toolNotes' : notes}
382
383
384 def tree_events (tree, start, label = '' ):
385 tool_calls = {c[ 'toolCallId' ]: c for c in tree.get( 'toolCalls' ) or []}
386 events, last_t = [], 0.0
387 for item in tree.get( 'timeline' ) or []:
388 if item.get( 'type' ) == 'tool-call' and item.get( 'toolCallId' ) in tool_calls:
389 c = tool_calls[item[ 'toolCallId' ]]
390 t = ts(c[ 'startedAt' ]) - start if c.get( 'startedAt' ) else last_t
391 last_t = t
392 result = c.get( 'result' )
393 events.append({
394 'kind' : 'tool' , 't' : t, 'name' : label + c[ 'toolName' ],
395 'duration' : (ts(c[ 'completedAt' ]) - ts(c[ 'startedAt' ])) if c.get( 'completedAt' ) and c.get( 'startedAt' ) else None ,
396 'args' : c.get( 'args' ), 'result' : result,
397 'error' : isinstance (result, dict ) and (result.get( 'success' ) is False or 'error' in result),
398 'note' : 'Confirmation card answered by the auto-approve policy.' if c.get( 'confirmation' ) else None ,
399 })
400 elif (item.get( 'content' ) or '' ).strip():
401 events.append({ 'kind' : 'reasoning' if item[ 'type' ] == 'reasoning' else 'text' , 't' : last_t,
402 'text' : item[ 'content' ], 'approx' : True })
403 for child in tree.get( 'children' ) or []:
404 events += tree_events(child, start, f ' { child.get( "role" ) or child.get( "agentId" ) } › ' )
405 return events
406
407
408 def load_instance_ai (run_dir, meta, prices):
409 trace = json.loads((run_dir / 'trace.json' ).read_text())
410 rich = trace[ 'rich' ].get( 'data' , trace[ 'rich' ])
411 messages = rich[ 'messages' ]
412 user = next (m for m in messages if m.get( 'role' ) == 'user' )
413 start = ts(user[ 'createdAt' ])
414 events = []
415 for m in messages:
416 if m.get( 'role' ) == 'assistant' and m.get( 'agentTree' ):
417 events += tree_events(m[ 'agentTree' ], start)
418 events.sort( key =lambda e: e[ 't' ])
419
420 calls, model, cost = [], None , 'not reported'
421 usage_file = run_dir / 'usage.json'
422 if usage_file.exists():
423 usage = json.loads(usage_file.read_text())
424 models = sorted ({u[ 'model' ] for u in usage if u.get( 'model' )})
425 model = ', ' .join(models) or None
426 calls = sorted ((
427 { 't' : ts(u[ 'start' ]) - start, 'start' : ts(u[ 'start' ]) - start,
428 'end' : ts(u[ 'end' ]) - start if u.get( 'end' ) else None ,
429 'name' : (u.get( 'name' ) or '' ).removeprefix( 'llm: ' ) or None ,
430 'prompt' : u[ 'in' ], 'cache_read' : u[ 'cache_read' ], 'cache_write' : u[ 'cache_write' ],
431 'out' : u[ 'out' ], 'cost' : u[ 'cost' ]} for u in usage), key =lambda c: c[ 't' ])
432 cost = f '$ { sum (c[ "cost" ] for c in calls) :.2f} '
433 duration = ts(meta[ 'finishedAt' ]) - ts(meta[ 'startedAt' ])
434 contexts = instance_ai_contexts(run_dir / 'context.json' )
435 first = next ( iter (contexts), None )
436 return { 'events' : events, 'calls' : calls, 'model' : model, 'duration' : f ' { duration :.0f} s' , 'cost' : cost,
437 'costSource' : 'langsmith' if usage_file.exists() else None ,
438 'context' : contexts.pop(first) if first else {
439 'messages' : [], 'tools' : None ,
440 'toolsNote' : 'This run has no context.json. Run langsmith_usage.py again to fetch it from LangSmith.' },
441 'agentContexts' : contexts}
442
443
444 LEADING_TAG_BLOCK = re.compile( r ' \s * < ([\w - ] + ) > . *? </ \1 >' , re.S)
445
446
447 def split_tagged_context (text):
448 """n8n puts tagged blocks, such as <thread-context>, before the user prompt. Returns (blocks, rest)."""
449 blocks, pos = [], 0
450 while m := LEADING_TAG_BLOCK .match(text, pos):
451 blocks.append(m.group( 0 ).strip())
452 pos = m.end()
453 return blocks, text[pos:].strip()
454
455
456 def instance_ai_contexts (path):
457 """{agent: context} from the context.json of langsmith_usage.py, in the order that the agents started.
458 The first agent answers the prompt, so the report shows its prompt in the User card, not here."""
459 if not path.exists():
460 return {}
461 contexts = {}
462 for i, agent in enumerate (json.loads(path.read_text())):
463 messages, system_seen = [], False
464 for m in agent.get( 'messages' ) or []:
465 if m[ 'role' ] == 'system' :
466 messages.append({ 'label' : 'System message' if system_seen else 'System prompt' , 'text' : m[ 'text' ]})
467 system_seen = True
468 continue
469 blocks, rest = split_tagged_context(m[ 'text' ])
470 messages += [{ 'label' : 'Context' , 'text' : b} for b in blocks]
471 if rest and i > 0 :
472 messages.append({ 'label' : 'Briefing' , 'text' : rest})
473 loaded = [{ ** t, 'group' : 'System · loaded with load_tool' } for t in agent.get( 'loadedTools' ) or []]
474 contexts[(agent[ 'agent' ] or '' ).removeprefix( 'llm: ' ) or None ] = {
475 'messages' : [m for m in messages if m[ 'text' ].strip()], 'tools' : (agent.get( 'tools' ) or []) + loaded or None }
476 return contexts
477
478
479 LOADERS = { 'instance-ai' : load_instance_ai, 'claude' : load_claude, 'codex' : load_codex}
480
481
482 def build_steps (calls, events):
483 """Group the trace events by the LLM call that produced them. Returns [{"call", "events"}]."""
484 if not calls:
485 return [{ 'call' : None , 'events' : [e]} for e in events]
486 steps = [{ 'call' : c, 'events' : []} for c in calls]
487 by_id = {c[ 'id' ]: s for c, s in zip (calls, steps) if c.get( 'id' )}
488 starts = [c.get( 'start' ) for c in calls]
489 ends = [c.get( 'end' ) if c.get( 'end' ) is not None else c[ 't' ] for c in calls]
490 known_starts = all (s is not None for s in starts)
491 last = len (calls) - 1
492 for e in events:
493 if e.get( 'call' ) in by_id:
494 by_id[e[ 'call' ]][ 'events' ].append(e)
495 continue
496 t = e[ 't' ]
497 # A tool call runs after the response that asked for it. Text is written during its response.
498 if e[ 'kind' ] == 'tool' :
499 edges = starts if known_starts else ends
500 i = max ((k for k, edge in enumerate (edges) if edge <= t + 0.05 ), default = 0 )
501 else :
502 edges = starts if known_starts else ends
503 i = next ((k for k, edge in enumerate (edges) if edge >= t - 0.05 ), last)
504 steps[i][ 'events' ].append(e)
505 return steps
506
507
508 def token_bar (c, max_prompt, max_out):
509 fresh = max (c[ 'prompt' ] - c[ 'cache_read' ] - c[ 'cache_write' ], 0 )
510 segs = '' .join(
511 f '<span class="seg { cls } " style="width: { 100 * n / max_prompt :.2f} %"></span>'
512 for cls , n in (( 'read' , c[ 'cache_read' ]), ( 'write' , c[ 'cache_write' ]), ( 'fresh' , fresh)) if n
513 )
514 return ( f '<div class="mini"><div class="bar"> { segs } </div>'
515 f '<div class="bar"><span class="seg outp" style="width: { 100 * c[ "out" ] / max_out :.2f} %"></span></div></div>' )
516
517
518 def step_preview (events, last):
519 tools = Counter(e[ 'name' ] for e in events if e[ 'kind' ] == 'tool' )
520 if tools:
521 return f '<code class="step-text"> { esc( ", " .join( f " { n } × { k } " if k > 1 else n for n, k in tools.items())) } </code>'
522 text = next ((e[ 'text' ] for kind in ( 'text' , 'reasoning' ) for e in events if e[ 'kind' ] == kind), None )
523 if text and last:
524 return '<span class="step-text">Final reply</span>'
525 if text:
526 return f '<span class="step-text"> { esc( " " .join(text.split())[: 160 ]) } </span>'
527 code = next ((e for e in events if e[ 'kind' ] == 'code' ), None )
528 if code:
529 return f '<code class="step-text"> { esc(code[ "name" ]) } : { esc( " " .join((code[ "code" ] or "" ).split())[: 160 ]) } </code>'
530 return '<span class="step-text dim">No text or tool calls</span>'
531
532
533 def size_label (text):
534 n = len ((text or '' ).encode())
535 return f ' { n } B' if n < 1024 else f ' { n / 1024 :.1f} KB'
536
537
538 def render_card (e, index):
539 time = f ' { e[ "t" ] :.1f} s { "*" if e.get( "approx" ) else "" } '
540 if e[ 'kind' ] == 'text' :
541 return ( f '<article class="card text"><div class="card-head"><span class="mono-label">Agent</span>'
542 f '<span class="mono-label"> { time } </span></div><div class="msg"> { esc(e[ "text" ]) } </div></article>' )
543 if e[ 'kind' ] == 'reasoning' :
544 return ( f '<details class="card reasoning"><summary class="card-head"><span class="mono-label">Reasoning</span>'
545 f '<span class="mono-label"> { time } </span></summary><div class="msg"> { esc(e[ "text" ]) } </div></details>' )
546 if e[ 'kind' ] == 'code' :
547 return ( f '<article class="card code"><div class="card-head"><span class="mono-label">Script</span>'
548 f '<code class="tool-name"> { esc(e[ "name" ]) } </code><span class="mono-label"> { time } </span></div>'
549 f '<pre> { esc(e[ "code" ] or "" ) } </pre>'
550 f '<details><summary>Output</summary><pre> { esc(pretty(e[ "result" ])) } </pre></details></article>' )
551 output = pretty(e[ 'result' ])
552 duration = f ' · { e[ "duration" ] :.1f} s' if e.get( 'duration' ) is not None else ''
553 status = '<span class="badge fail">Error</span>' if e[ 'error' ] else ''
554 note = f '<div class="note"> { esc(e[ "note" ]) } </div>' if e.get( 'note' ) else ''
555 return (
556 f '<article class="card tool { " is-error" if e[ "error" ] else "" } "><div class="card-head">'
557 f '<span class="mono-label">Tool # { index } </span><code class="tool-name"> { esc(e[ "name" ]) } </code> { status } '
558 f '<span class="mono-label"> { time }{ duration } </span></div> { note } '
559 f '<details open><summary>Input</summary><pre> { esc(pretty(e[ "args" ])) } </pre></details>'
560 f '<details { " open" if e[ "error" ] else "" } ><summary>Output · { size_label(output) } </summary>'
561 f '<pre> { esc(output) } </pre></details></article>'
562 )
563
564
565 def split_tool_name (name):
566 """('n8n', 'search_nodes') for 'mcp__n8n__search_nodes'. Names without an MCP prefix have no server."""
567 parts = (name or '' ).split( '__' )
568 return (parts[ 1 ], '__' .join(parts[ 2 :])) if len (parts) >= 3 and parts[ 0 ] == 'mcp' else ( None , name or '' )
569
570
571 def tool_group (tool):
572 """System for the built-in tools of the CLI, one group for each MCP server, and Other for the rest,
573 such as tools that the model provider runs."""
574 server = split_tool_name(tool[ 'name' ])[ 0 ]
575 if server:
576 return f 'MCP · { server } '
577 if tool.get( 'type' ) not in ( None , 'custom' , 'function' ):
578 return 'Other'
579 return 'System'
580
581
582 def schema_type (s):
583 if not isinstance (s, dict ):
584 return 'any'
585 if 'enum' in s:
586 values = [json.dumps(v) for v in s[ 'enum' ]]
587 return ' | ' .join(values[: 6 ]) + ( ' | …' if len (values) > 6 else '' )
588 if 'const' in s:
589 return json.dumps(s[ 'const' ])
590 for key in ( 'anyOf' , 'oneOf' ):
591 if key in s:
592 return ' | ' .join( dict .fromkeys(schema_type(x) for x in s[key]))
593 t = s.get( 'type' )
594 if isinstance (t, list ):
595 return ' | ' .join(t)
596 if t == 'array' :
597 return f ' { schema_type(s.get( "items" )) } []'
598 return t or 'any'
599
600
601 def schema_limits (s):
602 labels = (( 'minLength' , 'min length' ), ( 'maxLength' , 'max length' ), ( 'minimum' , 'min' ), ( 'maximum' , 'max' ),
603 ( 'minItems' , 'min items' ), ( 'maxItems' , 'max items' ), ( 'format' , 'format' ), ( 'pattern' , 'pattern' ),
604 ( 'default' , 'default' ))
605 return ' · ' .join( f ' { label } { json.dumps(s[k]) if k == "default" else s[k] } ' for k, label in labels if k in s)
606
607
608 def schema_rows (schema, prefix = '' , depth = 0 ):
609 """One row for each parameter. Nested objects add rows with a dotted name, two levels deep."""
610 if not isinstance (schema, dict ):
611 return []
612 required = set (schema.get( 'required' ) or [])
613 rows = []
614 for key, s in (schema.get( 'properties' ) or {}).items():
615 s = s if isinstance (s, dict ) else {}
616 rows.append({ 'name' : prefix + key, 'type' : schema_type(s), 'required' : key in required,
617 'description' : s.get( 'description' ) or '' , 'limits' : schema_limits(s), 'depth' : depth})
618 if depth < 2 :
619 items = s.get( 'items' ) if isinstance (s.get( 'items' ), dict ) else {}
620 if s.get( 'properties' ):
621 rows += schema_rows(s, f ' { prefix }{ key } .' , depth + 1 )
622 elif items.get( 'properties' ):
623 rows += schema_rows(items, f ' { prefix }{ key } [].' , depth + 1 )
624 return rows
625
626
627 def render_params (schema):
628 rows = schema_rows(schema)
629 if not rows:
630 return '<p class="dim">No parameters.</p>'
631 body = '' .join(
632 f '<tr><td style="padding-left: { 8 + 16 * r[ "depth" ] } px"><code> { esc(r[ "name" ]) } </code>'
633 + ( '<span class="req">required</span>' if r[ 'required' ] else '' )
634 + f '</td><td><code class="type"> { esc(r[ "type" ]) } </code></td><td> { esc(r[ "description" ]) } '
635 + ( f '<div class="limits"> { esc(r[ "limits" ]) } </div>' if r[ 'limits' ] else '' ) + '</td></tr>'
636 for r in rows
637 )
638 return f '<table class="params"><thead><tr><th>Parameter</th><th>Type</th><th>Description</th></tr></thead><tbody> { body } </tbody></table>'
639
640
641 def render_tool_spec (e):
642 if e.get( 'runtime' ):
643 return '<p class="dim">Not in the prompt. The model found this tool at run time, so the report has no schema for it.</p>'
644 if e.get( 'deferred' ) and not e.get( 'schema' ):
645 return ( '<p class="dim">Deferred. The model got only the name, and loads the schema with ToolSearch '
646 'when it needs the tool.</p>' )
647 if e.get( 'code' ):
648 return f '<div class="mono-label"> { esc(e[ "code" ][ "label" ]) } </div><pre> { esc(e[ "code" ][ "text" ]) } </pre>'
649 return ( f ' { render_params(e[ "schema" ]) } <details><summary>JSON schema</summary>'
650 f '<pre> { esc(json.dumps(e[ "schema" ], indent = 2 )) } </pre></details>' )
651
652
653 def render_tools (tools, events, notes = None ):
654 """Tool chips grouped by area, with the calls of this run, and the parameters of the selected tool.
655 notes replaces the text of an empty group."""
656 calls, failures = {}, {}
657 for e in events:
658 if e[ 'kind' ] in ( 'tool' , 'code' ):
659 name = e.get( 'fullName' ) or e[ 'name' ]
660 calls[name] = calls.get(name, 0 ) + 1
661 failures[name] = failures.get(name, 0 ) + bool (e.get( 'error' ))
662 entries = []
663 for t in tools:
664 short = split_tool_name(t[ 'name' ])[ 1 ]
665 count = calls.get(t[ 'name' ], 0 ) or calls.get(short, 0 )
666 failed = failures.get(t[ 'name' ], 0 ) or failures.get(short, 0 )
667 entries.append({ ** t, 'short' : short, 'group' : t.get( 'group' ) or tool_group(t), 'calls' : count, 'failed' : failed})
668 known = {t[ 'name' ] for t in tools}
669 for name, count in calls.items():
670 server, short = split_tool_name(name)
671 if server and name not in known:
672 entries.append({ 'name' : name, 'short' : short, 'description' : '' , 'group' : f 'MCP · { server } ' ,
673 'calls' : count, 'failed' : failures[name], 'runtime' : True })
674 used = [e for e in entries if e[ 'calls' ]]
675 selected = max (used, key =lambda e: e[ 'calls' ])[ 'name' ] if used else entries[ 0 ][ 'name' ]
676
677 names = {e[ 'group' ] for e in entries}
678 # System and MCP always show, so an empty group says that the model had none of these tools.
679 order = ([ 'System' ] + sorted (n for n in names if n.startswith( 'System ·' ))
680 + sorted (n for n in names if n.startswith( 'MCP' )) + ([ 'Other' ] if 'Other' in names else []))
681 if not any (n.startswith( 'MCP' ) for n in names):
682 order.insert(order.index( 'Other' ) if 'Other' in order else len (order), 'MCP' )
683 groups = []
684 for group in order:
685 members = sorted ((e for e in entries if e[ 'group' ] == group), key =lambda e: ( - e[ 'calls' ], e[ 'short' ]))
686 note = (notes or {}).get(group.split( ' · ' )[ 0 ])
687 if not members:
688 groups.append( f '<div class="tool-group empty"><div class="mono-label"> { esc(group) } · 0</div>'
689 f '<span class="dim"> { esc(note or "None given to the model." ) } </span></div>' )
690 continue
691 if all (e.get( 'runtime' ) for e in members):
692 note = note or 'Not in the prompt. The model found these tools at run time.'
693 elif any (e.get( 'deferred' ) and not e.get( 'schema' ) for e in members):
694 note = 'Dashed tools are deferred: the model got only their names, and loads a schema with ToolSearch.'
695 else :
696 note = None
697 chips = '' .join(
698 f '<button type="button" class="chip { " used" if e[ "calls" ] else "" } '
699 f ' { " runtime" if e.get( "runtime" ) or e.get( "deferred" ) else "" } '
700 f ' { " active" if e[ "name" ] == selected else "" } "'
701 f ' data-tool=" { esc(e[ "name" ]) } " data-search=" { esc((e[ "short" ] + " " + e[ "description" ][: 300 ]).lower()) } ">'
702 f ' { esc(e[ "short" ]) } '
703 + ( f '<span class="count">× { e[ "calls" ] } </span>' if e[ 'calls' ] else '' )
704 + ( f '<span class="fail-dot" title=" { e[ "failed" ] } failed"></span>' if e[ 'failed' ] else '' )
705 + '</button>'
706 for e in members
707 )
708 groups.append( f '<div class="tool-group"><div class="mono-label"> { esc(group) } · { len (members) } </div>'
709 + ( f '<span class="dim group-note"> { esc(note) } </span>' if note else '' )
710 + f '<div class="chips"> { chips } </div></div>' )
711
712 details = []
713 for e in entries:
714 usage = ''
715 if e[ 'calls' ]:
716 usage = f '<span class="badge partial">Called × { e[ "calls" ] } </span>'
717 if e[ 'failed' ]:
718 usage += f '<span class="step-flag"> { e[ "failed" ] } failed</span>'
719 details.append(
720 f '<div class="tool-detail" data-tool=" { esc(e[ "name" ]) } " { "" if e[ "name" ] == selected else " hidden" } >'
721 f '<div class="tool-detail-head"><code class="tool-name" title=" { esc(e[ "name" ]) } "> { esc(e[ "short" ]) } </code>'
722 f '<span class="mono-label"> { esc(e[ "group" ]) } </span> { usage } </div>'
723 + ( f '<div class="msg tool-desc"> { esc(e[ "description" ].strip()) } </div>' if e[ 'description' ].strip() else '' )
724 + f ' { render_tool_spec(e) } </div>'
725 )
726 return (
727 f '<details class="card context"><summary class="card-head"><span class="mono-label">Tools</span>'
728 f '<span class="mono-label"> { len (tools) } tools · { len (used) } called · { size_label(json.dumps(tools)) } </span></summary>'
729 f '<div class="tools-browser"><div class="tools-bar"><input type="search" class="tools-filter" '
730 f 'placeholder="Filter tools" aria-label="Filter tools">'
731 f '<span class="dim">Highlighted tools were called in this run.</span></div>'
732 f '<div class="tool-groups"> { "" .join(groups) } </div> { "" .join(details) } </div></details>'
733 )
734
735
736 def render_context (context, events):
737 """Collapsed cards for what the model gets before the prompt: system prompt, added messages and tools."""
738 if not context:
739 return []
740 cards = [
741 f '<details class="card context"><summary class="card-head"><span class="mono-label"> { esc(m[ "label" ]) } </span>'
742 f '<span class="mono-label"> { size_label(m[ "text" ]) } </span></summary><div class="msg"> { esc(m[ "text" ]) } </div></details>'
743 for m in context.get( 'messages' ) or []
744 ]
745 if context.get( 'tools' ):
746 cards.append(render_tools(context[ 'tools' ], events, context.get( 'toolNotes' )))
747 elif context.get( 'toolsNote' ):
748 cards.append( f '<article class="card context"><div class="card-head"><span class="mono-label">Tools</span>'
749 f '<span class="dim"> { esc(context[ "toolsNote" ]) } </span></div></article>' )
750 return cards
751
752
753 def sub_agent_events (events):
754 """Sub-agent tool events with the 'role › ' label removed, so that they match the tool names of the agent."""
755 return [{ ** e, 'name' : e[ 'name' ].split( ' › ' , 1 )[ 1 ], 'fullName' : None } for e in events
756 if e[ 'kind' ] == 'tool' and ' › ' in e[ 'name' ]]
757
758
759 def render_steps (calls, events, max_prompt, max_out, prompt, context = None , agent_contexts = None ):
760 steps = build_steps(calls, events)
761 if not steps:
762 return fold( 'LLM steps' , '' , '<p class="dim">No trace for this run.</p>' , cls = 'steps' )
763 names = {s[ 'call' ].get( 'name' ) for s in steps if s[ 'call' ]}
764 pending = dict (agent_contexts or {})
765 items, panes, tool_index = [], [], 0
766 for i, step in enumerate (steps, 1 ):
767 c, evs = step[ 'call' ], step[ 'events' ]
768 failed = sum ( 1 for e in evs if e[ 'kind' ] == 'tool' and e[ 'error' ])
769 # The first call answers the prompt, so the model context and the prompt open its detail.
770 # A sub-agent gets its own context, which opens the detail of its first call.
771 cards = render_context(context, events) if i == 1 else []
772 if c and c.get( 'name' ) in pending:
773 cards += render_context(pending.pop(c[ 'name' ]), sub_agent_events(events))
774 if i == 1 and prompt:
775 cards.append( f '<article class="card user"><div class="card-head"><span class="mono-label">User</span>'
776 f '<span class="mono-label">0.0s</span></div><div class="msg"> { esc(prompt) } </div></article>' )
777 for e in evs:
778 tool_index += e[ 'kind' ] == 'tool'
779 cards.append(render_card(e, tool_index))
780 t = c[ 't' ] if c else evs[ 0 ][ 't' ]
781 label = f '<span class="mono-label"> { esc(c[ "name" ]) } </span>' if c and c.get( 'name' ) and len (names) > 1 else ''
782 if failed:
783 label += f '<span class="step-flag"> { failed } failed</span>'
784 items.append(
785 f '<button type="button" class="step { " active" if i == 1 else "" }{ " is-error" if failed else "" } " data-step=" { i } ">'
786 f '<span class="step-top"><span class="num"> { i } </span> { step_preview(evs, i == len (steps)) } '
787 f '<span class="mono-label"> { t :.0f} s</span></span>'
788 + ( f '<span class="step-usage"> { label } </span>' if label else '' )
789 + (token_bar(c, max_prompt, max_out) if c else '' )
790 + '</button>'
791 )
792 stats = ''
793 if c:
794 fresh = max (c[ 'prompt' ] - c[ 'cache_read' ] - c[ 'cache_write' ], 0 )
795 cells = [( 'Input' , kilo(c[ 'prompt' ])), ( 'Cache read' , kilo(c[ 'cache_read' ])),
796 ( 'Cache write' , kilo(c[ 'cache_write' ])), ( 'Uncached' , kilo(fresh)), ( 'Output' , kilo(c[ 'out' ]))]
797 if c.get( 'cost' ) is not None :
798 cells.append(( 'Cost' , f '$ { c[ "cost" ] :.3f} ' ))
799 stats = '<dl class="stats">' + '' .join(
800 f '<div><dt class="mono-label"> { k } </dt><dd class="mono"> { v } </dd></div>' for k, v in cells) + '</dl>'
801 title = f 'Step { i } ' + ( f ' · { esc(c[ "name" ]) } ' if c and c.get( 'name' ) else '' )
802 panes.append(
803 f '<div class="step-detail" data-step=" { i } " { "" if i == 1 else " hidden" } >'
804 f '<div class="step-head"><h3> { title } </h3><span class="mono-label"> { t :.1f} s from the prompt</span></div>'
805 f ' { stats }{ "" .join(cards) or "<p class=dim>This call wrote no text and called no tools.</p>" } </div>'
806 )
807 legend = ( '<div class="legend mono-label"><span><i class="read"></i>Cache read</span><span><i class="write"></i>Cache write</span>'
808 '<span><i class="fresh"></i>Uncached input</span><span><i class="outp"></i>Output</span></div>' )
809 count = f ' { len (steps) } LLM calls' if calls else f ' { len (steps) } events, no per-call token data'
810 return fold( 'LLM steps' , f '<span class="mono-label"> { count } </span>' ,
811 f ' { legend if calls else "" } <div class="steps-layout"><div class="step-list"> { "" .join(items) } </div>'
812 f '<div class="step-pane"> { "" .join(panes) } </div></div>' , cls = 'steps' )
813
814
815 def code_spans (text):
816 """Show `backticked` parts of HTML text as code. Text with an odd number of backticks stays plain."""
817 if text.count( '`' ) % 2 :
818 return text
819 return '' .join( f '<code> { part } </code>' if i % 2 else part for i, part in enumerate (text.split( '`' )))
820
821
822 def inline_code (text):
823 """Escape text and show `backticked` parts as code."""
824 return code_spans(esc(text))
825
826
827 CHECK_GROUPS = (( False , 'bad' , '✗' , 'Not met' ), ( None , 'warn' , '?' , 'Not confirmed' ), ( True , 'ok' , '✓' , 'Met' ))
828
829
830 AUTO_CHECKS_FILE = 'auto-checks.json'
831 # A failed check in these groups of auto-checks.json changes the verdict, when a group has no gating flag.
832 GATING_GROUPS = ( 'correctness' , 'execution' , 'contract' )
833
834
835 def load_auto_checks (sub):
836 path = sub / AUTO_CHECKS_FILE
837 return json.loads(path.read_text()) if path.exists() else None
838
839
840 def load_workflow (sub):
841 path = sub / 'workflow.json'
842 return json.loads(path.read_text()) if path.exists() else None
843
844
845 def criteria_ids (expected):
846 return [num for _, items in parse_criteria(expected or '' ) or [] for num, _ in items]
847
848
849 def node_value (node, path):
850 """The value at a path such as parameters.text or rules.values[0]. A path without the parameters
851 prefix also reads from the parameters. Returns (found, value)."""
852 keys = [ int (k) if k.isdigit() else k for k in re.findall( r ' [ ^. \[\]\' " ] + ' , path or '' )]
853 for root in (node, node.get( 'parameters' ) or {}):
854 current, found = root, True
855 for key in keys:
856 if isinstance (current, list ) and isinstance (key, int ) and key < len (current):
857 current = current[key]
858 elif isinstance (current, dict ) and key in current:
859 current = current[key]
860 else :
861 found = False
862 break
863 if found:
864 return True , current
865 return False , None
866
867
868 def loose (text):
869 """Text for a containment test: one quote style and single spaces."""
870 return re.sub( r ' \s + ' , ' ' , text.replace( ' \\ "' , '"' ).replace( ' \\ n' , ' ' )).replace( '"' , "'" ).strip()
871
872
873 def evidence_problems (evidence, workflow):
874 """The evidence entries that the workflow does not contain. An entry is {node, path, value}.
875 An entry with "source": "message" quotes the final message and is not checked here."""
876 nodes = {n[ 'name' ]: n for n in (workflow or {}).get( 'nodes' , [])}
877 problems = []
878 for e in evidence:
879 if e.get( 'source' ) == 'message' :
880 continue
881 node = nodes.get(e.get( 'node' ))
882 if node is None :
883 problems.append( f 'no node is called " { e.get( "node" ) } "' )
884 continue
885 target = node
886 if e.get( 'path' ):
887 found, target = node_value(node, e[ 'path' ])
888 if not found:
889 problems.append( f ' { e[ "node" ] } has no { e[ "path" ] } ' )
890 continue
891 value = e.get( 'value' )
892 text = target if isinstance (target, str ) else json.dumps(target, ensure_ascii = False )
893 if value not in ( None , '' ) and loose( str (value)) not in loose(text):
894 where = f ' { e[ "node" ] } › { e[ "path" ] } ' if e.get( 'path' ) else e[ 'node' ]
895 problems.append( f ' { where } does not contain " { value } "' )
896 return problems
897
898
899 def is_strict (v):
900 """A panel verdict, or a verdict with cited evidence: a met check must then cite evidence."""
901 return v.get( 'version' , 1 ) >= 2 or bool (v.get( 'judges' )) or any (c.get( 'evidence' ) for c in v.get( 'checks' ) or [])
902
903
904 def add_note (check, note):
905 check[ 'note' ] = f ' { check[ "note" ] } { note } ' if check.get( 'note' ) else note
906
907
908 def verify_checks (v, workflow):
909 """The checks of a verdict, with a met check that has no evidence, or evidence that the
910 workflow does not contain, changed to not confirmed."""
911 strict = is_strict(v)
912 out = []
913 for c in v.get( 'checks' ) or []:
914 c = dict (c)
915 if c.get( 'ok' ) is True and strict:
916 if not c.get( 'evidence' ):
917 c[ 'ok' ] = None
918 add_note(c, 'No evidence cited, so this is not confirmed.' )
919 elif problems := evidence_problems(c[ 'evidence' ], workflow):
920 c[ 'ok' ] = None
921 add_note(c, 'Evidence not found: ' + '; ' .join(problems) + '.' )
922 out.append(c)
923 return out
924
925
926 def is_gating (group):
927 """Older auto-checks.json files have no gating flag, so their group key decides."""
928 return group.get( 'gating' , group[ 'key' ] in GATING_GROUPS )
929
930
931 def gating_failures (auto):
932 return [c for g in (auto or {}).get( 'groups' , []) if is_gating(g)
933 for c in g[ 'checks' ] if c.get( 'ok' ) is False and c.get( 'level' ) != 'info' ]
934
935
936 def brief_met (checks, ids):
937 """(met, total) over the expected outcome criteria. Without criteria ids, or when no check has
938 an id, over all checks."""
939 if not ids or all (c.get( 'id' ) is None for c in checks):
940 return sum ( 1 for c in checks if c.get( 'ok' ) is True ), len (checks)
941 by_id = {}
942 for c in checks:
943 by_id.setdefault( str (c.get( 'id' )), []).append(c.get( 'ok' ))
944 return sum ( 1 for i in ids if by_id.get(i) and all (ok is True for ok in by_id[i])), len (ids)
945
946
947 def compute_verdict (v, checks, auto, has_workflow, ids):
948 """pass, partial or fail from the criteria, the goal and the auto-checks. The judge verdict of an
949 old verdict file stays, but a failed auto-check turns its pass into partial."""
950 if not v:
951 return None
952 gating = gating_failures(auto)
953 if not is_strict(v):
954 return 'partial' if v.get( 'verdict' ) == 'pass' and gating else v.get( 'verdict' )
955 if not has_workflow or v.get( 'goalMet' ) is False :
956 return 'fail'
957 met, total = brief_met(checks, ids)
958 return 'pass' if total and met == total and not gating else 'partial'
959
960
961 def verdict_view (v, sub, expected):
962 """The verdict as the page shows it: verified checks and the computed verdict. Returns (view, auto)."""
963 auto = load_auto_checks(sub)
964 if not v:
965 return v, auto
966 workflow = load_workflow(sub)
967 checks = verify_checks(v, workflow)
968 view = dict (v, checks = checks, judgeVerdict = v.get( 'verdict' ))
969 view[ 'verdict' ] = compute_verdict(v, checks, auto, workflow is not None , criteria_ids(expected))
970 if quality := quality_group(v):
971 auto = dict (auto or { 'groups' : []}, groups = [ * (auto or {}).get( 'groups' , []), quality])
972 return view, auto
973
974
975 def quality_group (v):
976 """The LLM binary checks of the eval suite, as the judge panel answered them. They do not change the verdict."""
977 items = v.get( 'quality' ) or []
978 if not items:
979 return None
980 checks = [{ 'id' : f 'Q { i + 1 } ' , 'ok' : q.get( 'ok' ), 'text' : f ' { q[ "id" ] } : { q.get( "text" , "" ) } ' ,
981 'details' : [q[ 'note' ]] if q.get( 'note' ) else [],
982 'source' : { 'label' : 'Instance AI eval binary checks (LLM), answered by the judges' ,
983 'path' : 'packages/@n8n/instance-ai/evaluations/binaryChecks/checks' }} for i, q in enumerate (items)]
984 scored = [c for c in checks if c[ 'ok' ] is not None ]
985 return { 'key' : 'quality' , 'title' : 'Eval quality checks' , 'gating' : False ,
986 'note' : 'The LLM binary checks of the Instance AI eval suite. The judge panel answers them. This score does not change the verdict.' ,
987 'score' : { 'passed' : sum ( 1 for c in scored if c[ 'ok' ]), 'total' : len (scored)}, 'checks' : checks}
988
989
990 def standards_score (auto):
991 group = next ((g for g in (auto or {}).get( 'groups' , []) if g[ 'key' ] == 'standards' ), None )
992 return (group or {}).get( 'score' )
993
994
995 def verdict_head (v, expected = '' ):
996 checks = v.get( 'checks' ) or []
997 met, total = brief_met(checks, criteria_ids(expected))
998 count = f '<span class="mono-label"> { met } of { total } met</span>' if total else ''
999 panel = ''
1000 if v.get( 'judges' ):
1001 split = len (v.get( 'disagreements' ) or [])
1002 panel = ( f '<span class="mono-label { " err" if split else "" } "> { v[ "judges" ] } judges'
1003 + ( f ' · { split } split' if split else '' ) + '</span>' )
1004 return f '<span class="head-meta"> { panel }{ count }{ badge(v.get( "verdict" )) } </span>'
1005
1006
1007 CHECK_CLASS = { True : 'ok' , False : 'bad' , None : 'warn' }
1008 OTHER_FINDINGS = 'Other findings'
1009
1010
1011 def check_body (c):
1012 """The text of a judge check, with its note and the evidence that it cites."""
1013 parts = [ f '<span> { inline_code(c.get( "text" , "" )) } </span>' ]
1014 if c.get( 'note' ):
1015 parts.append( f '<span class="check-note"> { inline_code(c[ "note" ]) } </span>' )
1016 cited = []
1017 for e in c.get( 'evidence' ) or []:
1018 where = 'Final message' if e.get( 'source' ) == 'message' else ' › ' .join( filter ( None , [e.get( 'node' ), e.get( 'path' )]))
1019 value = f ' = <code> { esc( str (e[ "value" ])) } </code>' if e.get( 'value' ) not in ( None , '' ) else ''
1020 cited.append( f '<li> { esc(where) }{ value } </li>' )
1021 if cited:
1022 parts.append( f '<ul class="check-details"> { "" .join(cited) } </ul>' )
1023 return f '<span class="check-body"> { "" .join(parts) } </span>'
1024
1025
1026 def render_goal (v):
1027 """A failed goal shows above the checks, because it fails the run on its own. A minority of judges
1028 that says the goal is not met shows as a warning."""
1029 if v.get( 'goalMet' ) is not False :
1030 split = next ((d for d in v.get( 'disagreements' ) or [] if d.get( 'id' ) == 'goal' ), None )
1031 if not split:
1032 return ''
1033 no = split[ 'votes' ].count( False )
1034 reasons = '; ' .join(t for t in split.get( 'texts' ) or [] if t)
1035 return ( f '<ul class="checks goal"><li class="warn"><span class="mark" aria-hidden="true">?</span>'
1036 f '<span> { no } of { len (split[ "votes" ]) } judges { "says" if no == 1 else "say" } the workflow does not do what the prompt asks'
1037 f ' { ": " + inline_code(reasons) if reasons else "" } </span></li></ul>' )
1038 reason = f ': { inline_code(v[ "goalReason" ]) } ' if v.get( 'goalReason' ) else ''
1039 return ( f '<ul class="checks goal"><li class="bad"><span class="mark" aria-hidden="true">✗</span>'
1040 f '<span>The workflow does not do what the prompt asks { reason } </span></li></ul>' )
1041
1042
1043 def render_checks (v, expected = '' ):
1044 """The judge summary and the checks. expected is the plain expected outcome text.
1045 When it has numbered groups and the checks have an id, the checks show under the same groups as the brief.
1046 Otherwise the failed checks show first."""
1047 checks = v.get( 'checks' ) or []
1048 summary = f '<p class="summary"> { inline_code(v[ "summary" ]) } </p>' if v.get( 'summary' ) else ''
1049 summary += render_goal(v)
1050 criteria = parse_criteria(esc(expected)) if expected else None
1051 if criteria and any (c.get( 'id' ) is not None for c in checks):
1052 return summary + render_checks_by_criteria(checks, criteria)
1053 groups = []
1054 for ok, cls , mark, label in CHECK_GROUPS :
1055 items = [c for c in checks if c.get( 'ok' ) is ok]
1056 if items:
1057 rows = '' .join( f '<li class=" { cls } "><span class="mark" aria-hidden="true"> { mark } </span>'
1058 f ' { check_body(c) } </li>' for c in items)
1059 groups.append( f '<div class="check-group"><div class="mono-label"> { label } · { len (items) } </div>'
1060 f '<ul class="checks"> { rows } </ul></div>' )
1061 return summary + '' .join(groups)
1062
1063
1064 def render_checks_by_criteria (checks, criteria):
1065 """The checks under the groups of the expected outcome. A group with a failed or unconfirmed check shows open.
1066 A criterion without a check shows its own text as not judged. Checks with no known id show under Other findings."""
1067 by_id = {}
1068 for c in checks:
1069 by_id.setdefault( str (c.get( 'id' )), []).append(c)
1070 known = {num for _, items in criteria for num, _ in items}
1071 other = [c for c in checks if str (c.get( 'id' )) not in known]
1072 groups = [(heading, [(num, by_id.get(num), text) for num, text in items]) for heading, items in criteria]
1073 if other:
1074 groups.append(( OTHER_FINDINGS , [( '·' , [c], '' ) for c in other]))
1075 parts = []
1076 for heading, rows in groups:
1077 html_rows = '' .join(
1078 '' .join( f '<li class=" { CHECK_CLASS .get(c.get( "ok" ), "warn" ) } "><span class="mark num" aria-hidden="true"> { num } </span>'
1079 f ' { check_body(c) } </li>' for c in cs)
1080 if cs else f '<li class="warn"><span class="mark num" aria-hidden="true"> { num } </span>'
1081 f '<span class="dim">Not judged: { code_spans(text) } </span></li>'
1082 for num, cs, text in rows)
1083 body = f '<ul class="checks"> { html_rows } </ul>'
1084 if not heading:
1085 parts.append( f '<div class="criteria-group"> { body } </div>' )
1086 continue
1087 met = sum ( 1 for _, cs, _ in rows if cs and all (c.get( 'ok' ) is True for c in cs))
1088 failed = met < len (rows)
1089 count = f ' { len (rows) } found' if heading == OTHER_FINDINGS else f ' { met } of { len (rows) } met'
1090 label = f '<span class="mono-label { " err" if failed else "" } "> { count } </span>'
1091 parts.append( f '<details class="criteria-group" { " open" if failed else "" } ><summary class="criteria-title">'
1092 f '<span> { heading } </span> { label } </summary> { body } </details>' )
1093 return f '<div class="verdict-groups"> { "" .join(parts) } </div>'
1094
1095
1096 JUDGE_WAIT_TEXT = { 'queued' : 'Waiting for the judge.' , 'running' : 'The judge is checking this run.' }
1097
1098
1099 def fold (title, meta, body, open_ = False , cls = '' ):
1100 """A collapsible box: the title and meta HTML in the header row, the body below."""
1101 return ( f '<details class="box fold { " " + cls if cls else "" } " { " open" if open_ else "" } ><summary class="head">'
1102 f '<h2> { title } </h2> { meta } </summary> { body } </details>' )
1103
1104
1105 def render_verdict (v, expected = '' , judge = None , pending = False ):
1106 """The Expected outcome box. It shows open when the verdict is not pass or a check is not confirmed.
1107 pending is true on the live page, before the judges can run."""
1108 if not v:
1109 if pending:
1110 return fold( 'Expected outcome' , '<span class="mono-label">Not verified yet</span>' ,
1111 '<p class="dim">The judges check this run against the expected outcome after all builders finish.</p>' )
1112 meta = pending_badge( 'judge' , judge) or '<span class="mono-label">Not judged</span>'
1113 return fold( 'Expected outcome' , meta, f '<p class="dim"> { JUDGE_WAIT_TEXT .get(judge, "Not judged yet." ) } </p>' )
1114 attention = v.get( 'verdict' ) != 'pass' or any (c.get( 'ok' ) is not True for c in v.get( 'checks' ) or [])
1115 return fold( 'Expected outcome' , verdict_head(v, expected), render_checks(v, expected), open_ = attention)
1116
1117
1118 def auto_check_row (c):
1119 info = c.get( 'level' ) == 'info'
1120 cls = 'info' if info else CHECK_CLASS .get(c.get( 'ok' ), 'warn' )
1121 details = '' .join( f '<li> { inline_code(d) } </li>' for d in c.get( 'details' ) or [])
1122 source = c.get( 'source' ) or {}
1123 links = []
1124 for s in (source, source.get( 'also' ) or {}):
1125 if s.get( 'url' ):
1126 links.append( f '<a href=" { esc(s[ "url" ]) } " target="_blank"> { esc(s[ "label" ]) } ↗</a>' )
1127 elif s.get( 'label' ):
1128 links.append(esc(s[ 'label' ]) + ( f ' (<code> { esc(s[ "path" ]) } </code>)' if s.get( 'path' ) else '' ))
1129 return ( f '<li class=" { cls } "><span class="mark num" aria-hidden="true"> { esc(c[ "id" ]) } </span><span class="check-body">'
1130 f '<span> { inline_code(c.get( "text" , "" )) } </span>'
1131 + ( f '<ul class="check-details"> { details } </ul>' if details else '' )
1132 + ( f '<span class="check-source">Source: { " · " .join(links) } </span>' if links else '' )
1133 + '</span></li>' )
1134
1135
1136 def render_auto_checks (auto):
1137 """The groups of auto-checks.json. A gating group with a failed check shows open."""
1138 if not auto:
1139 return ''
1140 parts = []
1141 for g in auto.get( 'groups' , []):
1142 judged = [c for c in g[ 'checks' ] if c.get( 'ok' ) is not None and c.get( 'level' ) != 'info' ]
1143 failed = sum ( 1 for c in judged if c[ 'ok' ] is False )
1144 gating = is_gating(g)
1145 score = g.get( 'score' )
1146 if score:
1147 label = f 'Score { score[ "passed" ] } of { score[ "total" ] } · does not change the verdict'
1148 else :
1149 label = f ' { len (judged) - failed } of { len (judged) } passed' if judged else 'Not checked'
1150 rows = '' .join(auto_check_row(c) for c in g[ 'checks' ])
1151 parts.append(fold(
1152 esc(g[ 'title' ]), f '<span class="mono-label { " err" if gating and failed else "" } "> { esc(label) } </span>' ,
1153 f '<p class="dim auto-note"> { esc(g.get( "note" , "" )) } </p><ul class="checks"> { rows } </ul>' ,
1154 open_ = gating and failed > 0 , cls = 'auto-group' ))
1155 return '' .join(parts)
1156
1157
1158 PENDING_CHECK_GROUPS = (
1159 ( 'Correctness' , 'Checks from the node definitions, the n8n editor and the workflow graph.' ),
1160 ( 'Execution with pinned data' , 'Runs the workflow once for each scenario, with pinned data.' ),
1161 ( 'n8n standards' , 'Rules from n8n-io/skills and the workflow-sdk best practices.' ),
1162 )
1163
1164
1165 def render_pending_checks ():
1166 """Placeholders for the auto-check groups that every run gets, before auto-checks.json exists."""
1167 return '' .join(fold(title, '<span class="mono-label">Not run yet</span>' ,
1168 f '<p class="dim auto-note"> { esc(note) } These checks run after all builders finish.</p>' ,
1169 cls = 'auto-group' ) for title, note in PENDING_CHECK_GROUPS )
1170
1171
1172 def render_result (verdict, judge, auto):
1173 """The Result cell: the verdict badge and the standards score."""
1174 score = standards_score(auto)
1175 std = ( f ' <span class="mono-label std" title="n8n standards score. It does not change the verdict.">'
1176 f 'Std { score[ "passed" ] } / { score[ "total" ] } </span>' if score and score[ 'total' ] else '' )
1177 return badge((verdict or {}).get( 'verdict' ), judge) + std
1178
1179
1180 PLAYER_CACHE = ASSET_CACHE / f 'asciinema-player- { PLAYER_VERSION } '
1181 PLAYER_CDN = f 'https://cdn.jsdelivr.net/npm/asciinema-player@ { PLAYER_VERSION } /dist/bundle/'
1182
1183
1184 def ensure_player_cache ():
1185 """Download the asciinema player once. Raises OSError when the download fails."""
1186 PLAYER_CACHE .mkdir( parents = True , exist_ok = True )
1187 for name in PLAYER_FILES :
1188 if not ( PLAYER_CACHE / name).exists():
1189 partial = PLAYER_CACHE / f ' { name } .part'
1190 urllib.request.urlretrieve( PLAYER_CDN + name, partial)
1191 partial.rename( PLAYER_CACHE / name)
1192
1193
1194 def player_assets (report):
1195 """Copy the asciinema player next to the page, so the page works offline. Returns its URLs."""
1196 cache, cdn = PLAYER_CACHE , PLAYER_CDN
1197 try :
1198 ensure_player_cache()
1199 (report / 'vendor' ).mkdir( exist_ok = True )
1200 for name in PLAYER_FILES :
1201 shutil.copy(cache / name, report / 'vendor' / name)
1202 return [ f 'vendor/ { name } ' for name in PLAYER_FILES ]
1203 except OSError :
1204 return [cdn + name for name in PLAYER_FILES ]
1205
1206
1207 def cast_length (text):
1208 """Seconds from the start of an asciicast to its last frame."""
1209 for line in reversed (text.splitlines()):
1210 if line.startswith( '[' ):
1211 try :
1212 return float (json.loads(line)[ 0 ])
1213 except ( ValueError , IndexError , TypeError ):
1214 continue
1215 return 0.0
1216
1217
1218 def collect_screens (runs, run_dir, report, verdicts):
1219 """Copy the screen recordings next to the page. Returns one entry for each recording."""
1220 screens_dir = report / 'screens'
1221 shutil.rmtree(screens_dir, ignore_errors = True )
1222 screens = []
1223 for name, meta, data in runs:
1224 source = run_dir / name / meta[ 'screen' ] if meta.get( 'screen' ) else None
1225 if not source or not source.exists():
1226 continue
1227 screens_dir.mkdir( parents = True , exist_ok = True )
1228 screen = { 'name' : name, 'title' : TITLES .get(meta[ 'kind' ], meta[ 'kind' ]), 'duration' : data[ 'duration' ],
1229 'verdict' : (verdicts.get(name) or {}).get( 'verdict' )}
1230 if source.suffix == '.cast' :
1231 text = source.read_text()
1232 # Browsers block fetch() on file:// pages, so the recording is loaded as a script.
1233 (screens_dir / f ' { name } .cast.js' ).write_text(
1234 f '(window.CASTS = window.CASTS || {{}} )[ { json.dumps(name) } ] = { json.dumps(text) } ; \n ' )
1235 screen.update( type = 'cast' , src = f 'screens/ { name } .cast.js' , end = cast_length(text))
1236 else :
1237 shutil.copy(source, screens_dir / f ' { name }{ source.suffix } ' )
1238 screen.update( type = 'video' , src = f 'screens/ { name }{ source.suffix } ' )
1239 poster = source.with_name( f ' { source.stem } -poster.png' )
1240 if poster.exists():
1241 shutil.copy(poster, screens_dir / f ' { name } -poster.png' )
1242 screen[ 'poster' ] = f 'screens/ { name } -poster.png'
1243 screens.append(screen)
1244 return screens
1245
1246
1247 PLAY_ICON = '<span class="play-icon"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M8 5v14l11-7z"/></svg></span>'
1248
1249
1250 REPLAY_FINEPRINT = ( 'Terminal screens are exact recordings of the interactive CLI. Instance AI is a video '
1251 'of the n8n UI. Skip pauses shortens idle time in the terminal recordings only. Press Esc to close.' )
1252
1253
1254 def screen_kind (s):
1255 return esc(s.get( 'label' ) or ( 'Terminal' if s[ 'type' ] == 'cast' else 'n8n UI' ))
1256
1257
1258 def render_replay (screens, fineprint = REPLAY_FINEPRINT ):
1259 """A strip of final screens that play in place, and a full-screen player that Expand opens.
1260
1261 A screen can set "label" to replace the "Terminal" or "n8n UI" text under its card."""
1262 if not screens:
1263 return ''
1264 cards = []
1265 for s in screens:
1266 if s[ 'type' ] == 'cast' :
1267 thumb = f '<div class="thumb-player" data-cast=" { esc(s[ "name" ]) } "></div>'
1268 elif s.get( 'poster' ):
1269 thumb = f '<img src=" { esc(s[ "poster" ]) } " alt="">'
1270 else :
1271 thumb = f '<video muted preload="metadata" src=" { esc(s[ "src" ]) } "></video>'
1272 kind = screen_kind(s)
1273 cards.append(
1274 f '<div class="reel-card"><div class="thumb" data-thumb=" { esc(s[ "name" ]) } ">'
1275 f '<button type="button" class="thumb-poster" data-play=" { esc(s[ "name" ]) } " aria-label="Play { esc(s[ "title" ]) } ">'
1276 f ' { thumb }{ PLAY_ICON } </button></div>'
1277 f '<div class="reel-meta"><span class="run-name"> { esc(s[ "title" ]) } </span> { badge(s[ "verdict" ]) if s[ "verdict" ] else "" } </div>'
1278 f '<span class="mono-label"> { kind } · { esc(s[ "duration" ]) } </span></div>'
1279 )
1280 side_by_side = '<button type="button" class="btn accent" data-open="all">Expand</button>'
1281 tabs = '' .join( f '<button type="button" class="tab" data-replay=" { esc(s[ "name" ]) } "> { esc(s[ "title" ]) } </button>'
1282 for s in screens)
1283 if len (screens) > 1 :
1284 tabs += '<button type="button" class="tab" data-replay="all">Side by side</button>'
1285 figures = '' .join(
1286 f '<figure class="screen" data-run=" { esc(s[ "name" ]) } "><figcaption class="screen-head">'
1287 f '<span class="run-name"> { esc(s[ "title" ]) } </span><span class="mono-label"> { screen_kind(s) } · { esc(s[ "duration" ]) } </span>'
1288 f ' { badge(s[ "verdict" ]) if s.get( "verdict" ) else "" } </figcaption><div class="screen-frame">'
1289 + ( f '<video controls muted playsinline preload="metadata" src=" { esc(s[ "src" ]) } "'
1290 + ( f ' poster=" { esc(s[ "poster" ]) } "' if s.get( 'poster' ) else '' ) + '></video>' if s[ 'type' ] == 'video'
1291 else '<div class="player"></div>' )
1292 + '</div></figure>'
1293 for s in screens
1294 )
1295 speeds = '' .join( f '<button type="button" data-speed=" { n } " aria-pressed=" { str (n == 1 ).lower() } "> { n } ×</button>'
1296 for n in ( 1 , 2 , 4 , 8 ))
1297 return (
1298 f '<div class="box replay"><div class="head"><h2>Replay</h2> { side_by_side } </div>'
1299 f '<div class="reel"> { "" .join(cards) } </div></div>'
1300 '<div class="theater" id="theater" role="dialog" aria-modal="true" aria-label="Replay" hidden>'
1301 f '<div class="theater-bar"><h2>Replay</h2><div class="tabs"> { tabs } </div><div class="theater-controls">'
1302 f '<div class="speed" id="speed" role="group" aria-label="Speed"><span class="mono-label">Speed</span> { speeds } </div>'
1303 '<label class="mono-label"><input type="checkbox" id="skip-idle" checked> Skip pauses</label>'
1304 '<button type="button" class="btn" id="restart-all">Restart</button>'
1305 '<button type="button" class="btn accent" id="play-all">Play all</button>'
1306 '<button type="button" class="btn" id="theater-close">Close</button></div></div>'
1307 f '<div class="replay-grid" id="replay-grid"> { figures } </div>'
1308 f '<p class="fineprint"> { fineprint } </p></div>'
1309 )
1310
1311
1312 PENDING_BADGES = {
1313 'judge' : { 'queued' : ( 'st-waiting' , 'Judge queued' ), 'running' : ( 'st-running' , 'Judging' )},
1314 'prices' : { 'queued' : ( 'st-waiting' , 'Price queued' ), 'running' : ( 'st-running' , 'Finding price' )},
1315 }
1316
1317
1318 def pending_badge (step, state):
1319 """The badge of a step that comes after the run, or '' when the step is not queued or running."""
1320 cls , label = PENDING_BADGES [step].get(state, ( None , None ))
1321 return f '<span class="badge { cls } "> { label } </span>' if cls else ''
1322
1323
1324 def badge (verdict, judge = None ):
1325 """The verdict badge. Without a verdict, judge is the state of the judge step."""
1326 if not verdict:
1327 return pending_badge( 'judge' , judge) or '<span class="badge none">Not judged</span>'
1328 return f '<span class="badge { esc(verdict) } "> { esc(verdict.upper()) } </span>'
1329
1330
1331 RUN_ORDER = { 'instance-ai' : 0 , 'claude' : 1 , 'codex' : 2 }
1332
1333
1334 def load_case (run_dir):
1335 case = json.loads((run_dir / 'case.json' ).read_text())
1336 if 'prompt' not in case:
1337 case [ 'prompt' ] = (run_dir / 'prompt.txt' ).read_text().strip()
1338 return case
1339
1340
1341 def load_verdicts (run_dir):
1342 verdicts_file = run_dir / 'verdicts.json'
1343 return json.loads(verdicts_file.read_text()) if verdicts_file.exists() else {}
1344
1345
1346 def load_run (sub, meta, prices):
1347 try :
1348 return LOADERS [meta[ 'kind' ]](sub, meta, prices)
1349 except Exception as e: # keep the other runs when one run has broken files
1350 return { 'events' : [], 'calls' : [], 'model' : meta.get( 'model' ), 'duration' : '-' , 'cost' : '-' ,
1351 'costSource' : None , 'loadError' : f ' { type (e). __name__ } : { e } ' }
1352
1353
1354 def collect_runs (run_dir, case):
1355 runs = []
1356 for meta_file in sorted (run_dir.glob( '*/run.json' )):
1357 sub = meta_file.parent
1358 meta = json.loads(meta_file.read_text())
1359 if meta.get( 'kind' ) not in LOADERS or sub.name == 'report' :
1360 continue
1361 runs.append((sub.name, meta, load_run(sub, meta, case.get( 'prices' ))))
1362 runs.sort( key =lambda r: ( RUN_ORDER .get(r[ 1 ][ 'kind' ], 9 ), r[ 0 ]))
1363 return runs
1364
1365
1366 def token_scale (runs):
1367 all_calls = [c for _, _, d in runs for c in d[ 'calls' ]]
1368 return max ([c[ 'prompt' ] for c in all_calls] + [ 1 ]), max ([c[ 'out' ] for c in all_calls] + [ 1 ])
1369
1370
1371 def run_title (name, meta):
1372 title = TITLES .get(meta[ 'kind' ], meta[ 'kind' ])
1373 return title if name in ( 'iai' , 'claude' , 'codex' ) else f ' { title } ( { name } )'
1374
1375
1376 def render_row_cells (name, meta, data, result):
1377 """The table cells of one run. result is the HTML of the Result cell.
1378 data can have costPending, the state of the price step, and costTitle, the source of the cost."""
1379 calls = data[ 'calls' ]
1380 title = f ' title=" { esc(data[ "costTitle" ]) } "' if data.get( 'costTitle' ) else ''
1381 cost = pending_badge( 'prices' , data.get( 'costPending' )) or f '<strong { title } > { esc(data[ "cost" ]) } </strong>'
1382 tools = [e for e in data[ 'events' ] if e[ 'kind' ] == 'tool' ]
1383 errors = sum (e[ 'error' ] for e in tools)
1384 prompt = sum (c[ 'prompt' ] for c in calls)
1385 cached = sum (c[ 'cache_read' ] for c in calls)
1386 out = sum (c[ 'out' ] for c in calls)
1387 cells = [
1388 f '<span class="run-name"> { esc(run_title(name, meta)) } </span>' , esc(data.get( 'model' ) or '-' ),
1389 ( f '<span title=" { esc( "Source: " + meta[ "effortSource" ]) } "> { esc(meta[ "effort" ]) } </span>'
1390 if meta.get( 'effort' ) and meta.get( 'effortSource' ) else esc(meta.get( 'effort' ) or '-' )),
1391 result, data[ 'duration' ], str ( len (calls)),
1392 f ' { len (tools) } ' + ( f ' <span class="err">( { errors } err)</span>' if errors else '' ),
1393 kilo(prompt), f ' { 100 * cached / prompt :.0f} %' if prompt else '-' , kilo(out), cost,
1394 ]
1395 return '' .join( f '<td> { c } </td>' for c in cells)
1396
1397
1398 def render_canvas_box (name, meta, report, base_url, empty = 'No workflow was created.' ):
1399 wf = meta.get( 'workflowId' )
1400 image = report / 'img' / f ' { name } .png'
1401 # capture_canvas.mjs takes the screenshot at twice the CSS size.
1402 size = ' ' .join( f ' { k } =" { v // 2 } "' for k, v in zip (( 'width' , 'height' ), png_size(image))) if image.exists() else ''
1403 n8n_url = f ' { base_url } /workflow/ { wf } ' if wf else ''
1404 title = esc(run_title(name, meta))
1405 canvas = (
1406 f '<a class="canvas" href="img/ { name } .png" target="_blank" data-canvas=" { name } " data-title=" { title } " '
1407 f 'data-n8n=" { esc(n8n_url) } "><img src="img/ { name } .png" { size } alt=" { title } canvas"></a>'
1408 if image.exists() else f '<p class="dim"> { esc(empty) } </p>'
1409 )
1410 workflow = load_workflow(report.parent / name)
1411 nodes = [n for n in (workflow or {}).get( 'nodes' , []) if n.get( 'type' ) != 'n8n-nodes-base.stickyNote' ]
1412 count = f '<span class="mono-label"> { len (nodes) } node { "" if len (nodes) == 1 else "s" } </span>' if workflow else ''
1413 link = f '<a href=" { n8n_url } " target="_blank">Open in n8n ↗</a>' if wf else ''
1414 return fold( 'Canvas' , f '<span class="head-meta"> { count }{ link } </span>' , canvas, cls = 'canvas-box' )
1415
1416
1417 def render_panel_body (data, verdict, canvas_box, max_prompt, max_out, prompt, expected = '' , judge = None , auto = None ,
1418 pending = False ):
1419 """The content of the section of one run: canvas, result, auto-checks and LLM steps.
1420 judge is the state of the judge step, and auto the content of auto-checks.json. pending is true on the
1421 live page: checks and verdicts that do not exist yet show as not run."""
1422 error = f '<div class="box err">Could not read this run: { esc(data[ "loadError" ]) } </div>' if data.get( 'loadError' ) else ''
1423 checks = render_pending_checks() if pending and not auto else render_auto_checks(auto)
1424 return ( f ' { error } <div class="folds"> { canvas_box }{ render_verdict(verdict, expected, judge, pending) } '
1425 f ' { checks } </div>'
1426 f ' { render_steps(data[ "calls" ], data[ "events" ], max_prompt, max_out, prompt, data.get( "context" ), data.get( "agentContexts" )) } ' )
1427
1428
1429 POST_RUN_STEPS = ( 'judge' , 'prices' )
1430 POST_RUN_STATES = ( 'queued' , 'running' , 'done' )
1431
1432
1433 def step_arg (value):
1434 step, _, state = value.partition( '=' )
1435 if step not in POST_RUN_STEPS or state not in POST_RUN_STATES :
1436 raise argparse.ArgumentTypeError( f 'use { "|" .join( POST_RUN_STEPS ) } = { "|" .join( POST_RUN_STATES ) } , not { value } ' )
1437 return step, state
1438
1439
1440 def read_status (run_dir, steps = ()):
1441 """status.json. steps are (step, state) pairs to write to its postRun object first."""
1442 status_file = run_dir / 'status.json'
1443 status = json.loads(status_file.read_text()) if status_file.exists() else {}
1444 if steps:
1445 status.setdefault( 'postRun' , {}).update(steps)
1446 tmp = status_file.with_suffix( '.tmp' )
1447 tmp.write_text(json.dumps(status, indent = 2 ))
1448 os.replace(tmp, status_file)
1449 return status
1450
1451
1452 def cost_title (meta, data, prices):
1453 """Where the cost of a run comes from, for the tooltip of its Cost cell."""
1454 source = data.get( 'costSource' )
1455 if source == 'langsmith' :
1456 return 'From the LangSmith call costs.'
1457 if source == 'claude-code' :
1458 return 'The estimate of Claude Code.'
1459 p = prices.get(data.get( 'model' )) if source == 'prices' else None
1460 if not p:
1461 return 'Instance AI cost comes from LangSmith.' if meta[ 'kind' ] == 'instance-ai' else 'No published price.'
1462 return ( f 'Tokens × the published price of { data[ "model" ] } : $ { p.get( "input" , 0 ) } input, '
1463 f '$ { p.get( "cached" , 0 ) } cache read, $ { p.get( "cacheWrite" , p.get( "input" , 0 )) } cache write, '
1464 f '$ { p.get( "output" , 0 ) } output per million tokens.' + ( f ' Source: { p[ "source" ] } ' if p.get( 'source' ) else '' ))
1465
1466
1467 RELOAD_JS = """
1468 // Served by the live server while a step is pending: reload when the page is built again.
1469 window.addEventListener('DOMContentLoaded', ()=>{
1470 const saved = JSON.parse(sessionStorage.getItem('report-view') || 'null');
1471 if (saved) {
1472 sessionStorage.removeItem('report-view');
1473 if (document.getElementById(saved.run)) show(saved.run);
1474 window.scrollTo(0, saved.y);
1475 }
1476 if (!PENDING || !location.protocol.startsWith('http')) return;
1477 setInterval(async ()=>{
1478 try {
1479 const version = (await (await fetch('version.txt', {cache: 'no-store'})).text()).trim();
1480 if (version && version !== REPORT_VERSION) {
1481 sessionStorage.setItem('report-view', JSON.stringify({run: document.querySelector('.run.active')?.id, y: window.scrollY}));
1482 location.reload();
1483 }
1484 } catch {}
1485 }, 3000);
1486 });
1487 """
1488
1489
1490 def main ():
1491 ap = argparse.ArgumentParser( description = 'Build the comparison page of a run directory.' )
1492 ap.add_argument( 'run_dir' , nargs = '?' , default = '.' )
1493 ap.add_argument( '--step' , type = step_arg, action = 'append' , default = [],
1494 help = 'Set the state of a step after the run, for example judge=running. Repeat for more steps.' )
1495 args = ap.parse_args()
1496 run_dir = Path(args.run_dir).resolve()
1497 status = read_status(run_dir, args.step)
1498 case = load_case(run_dir)
1499 verdicts = load_verdicts(run_dir)
1500 report = run_dir / 'report'
1501 (report / 'img' ).mkdir( parents = True , exist_ok = True )
1502 if ASSET_CACHE .exists():
1503 for sub in ( 'fonts' , 'brand' ):
1504 if ( ASSET_CACHE / sub).exists():
1505 shutil.copytree( ASSET_CACHE / sub, report / sub, dirs_exist_ok = True )
1506
1507 runs = collect_runs(run_dir, case)
1508 if not runs:
1509 sys.exit( 'No runs found.' )
1510 for name, _, _ in runs:
1511 if (run_dir / name / 'canvas.png' ).exists():
1512 shutil.copy(run_dir / name / 'canvas.png' , report / 'img' / f ' { name } .png' )
1513
1514 max_prompt, max_out = token_scale(runs)
1515 base_url = case.get( 'baseUrl' , 'http://localhost:5678' )
1516 post_run = status.get( 'postRun' ) or {}
1517 # A run with a verdict is judged, so a step that was not marked done does not show as pending.
1518 judge = post_run.get( 'judge' ) if any (name not in verdicts for name, _, _ in runs) else None
1519 pending = False
1520 views = {name: verdict_view(verdicts.get(name), run_dir / name, case.get( 'expected' ) or '' ) for name, _, _ in runs}
1521
1522 rows, panels = [], []
1523 for i, (name, meta, data) in enumerate (runs):
1524 active = ' active' if i == 0 else ''
1525 verdict, auto = views[name]
1526 data[ 'costTitle' ] = cost_title(meta, data, case.get( 'prices' ) or {})
1527 if meta[ 'kind' ] != 'instance-ai' and data.get( 'costSource' ) is None :
1528 data[ 'costPending' ] = post_run.get( 'prices' )
1529 pending = pending or ( not verdict and judge in ( 'queued' , 'running' )) \
1530 or data.get( 'costPending' ) in ( 'queued' , 'running' )
1531 rows.append( f '<tr class="row { active } " data-run=" { name } " onclick="show( \'{ name }\' )">'
1532 + render_row_cells(name, meta, data, render_result(verdict, judge, auto)) + '</tr>' )
1533 canvas_box = render_canvas_box(name, meta, report, base_url)
1534 panels.append( f '<section class="run { active } " id=" { name } ">'
1535 + render_panel_body(data, verdict, canvas_box, max_prompt, max_out, case[ 'prompt' ],
1536 case.get( 'expected' ) or '' , judge, auto = auto)
1537 + '</section>' )
1538
1539 # The agent reads this file to find the models that still need a published price.
1540 models = [{ 'run' : name, 'kind' : meta[ 'kind' ], 'model' : data.get( 'model' ), 'costSource' : data.get( 'costSource' )}
1541 for name, meta, data in runs]
1542 (run_dir / 'models.json' ).write_text(json.dumps(models, indent = 2 ))
1543 unpriced = sorted ({m[ 'model' ] for m in models
1544 if m[ 'model' ] and m[ 'kind' ] != 'instance-ai' and m[ 'costSource' ] != 'prices' })
1545
1546 screens = collect_screens(runs, run_dir, report, {name: v for name, (v, _) in views.items() if v})
1547 head, scripts = '' , ''
1548 if any (s[ 'type' ] == 'cast' for s in screens):
1549 player_js, player_css = player_assets(report)
1550 head = f '<link rel="stylesheet" href=" { esc(player_css) } ">'
1551 scripts = f '<script src=" { esc(player_js) } "></script>' + '' .join(
1552 f '<script src=" { esc(s[ "src" ]) } "></script>' for s in screens if s[ 'type' ] == 'cast' )
1553 version = str (time.time_ns())
1554 scripts += ( f '<script>const SCREENS = { json.dumps(screens) } ; const REPORT_VERSION = " { version } "; '
1555 f 'const PENDING = { json.dumps(pending) } ;</script><script> { RELOAD_JS } </script>' )
1556
1557 meta = [ f ' { len (runs) } builder' + ( 's' if len (runs) > 1 else '' )]
1558 if status.get( 'startedAt' ):
1559 meta.insert( 0 , datetime.fromtimestamp(status[ 'startedAt' ]).strftime( ' %b %-d , %Y · %H:%M' ))
1560 if status.get( 'mode' ) == 'parallel' :
1561 meta.append( 'run in parallel' )
1562 page = fill_template(esc(case.get( 'title' ) or 'Builder comparison' ), '' .join(rows), '' ,
1563 '' .join(panels), replay = render_replay(screens), head = head, scripts = scripts,
1564 prompt = esc(case[ 'prompt' ]), expected = esc(case.get( 'expected' ) or '-' ),
1565 meta = esc( ' · ' .join(meta)))
1566 (report / 'index.html' ).write_text(page)
1567 (report / 'version.txt' ).write_text(version)
1568 print (report / 'index.html' )
1569 if unpriced:
1570 print ( 'models without a published price in case.json: ' + ', ' .join(unpriced))
1571
1572
1573 LIST_ITEM = re.compile( r ' ^\s * (?:(\d + )[ .) ] | [ -*• ])\s + ' )
1574
1575
1576 def parse_criteria (expected):
1577 """Split an expected outcome into [(heading, [(number, text)])].
1578 A line that is not a list item starts a new group. Returns None when the text is not a list."""
1579 groups = []
1580 for line in expected.splitlines():
1581 if not line.strip():
1582 continue
1583 match = LIST_ITEM .match(line)
1584 if not match:
1585 groups.append((line.strip().rstrip( ':' ), []))
1586 continue
1587 if not groups:
1588 groups.append(( '' , []))
1589 items = groups[ - 1 ][ 1 ]
1590 items.append((match.group( 1 ) or str ( sum ( len (g[ 1 ]) for g in groups)), line[match.end():].strip()))
1591 if sum ( len (items) for _, items in groups) < 2 or any ( not items for _, items in groups):
1592 return None
1593 return groups
1594
1595
1596 def render_criteria (groups):
1597 """The groups from parse_criteria as numbered lists. A group with a heading shows collapsed. The text is HTML."""
1598 parts = []
1599 for heading, items in groups:
1600 rows = '' .join( f '<li><span class="num"> { num } </span><span> { code_spans(text) } </span></li>' for num, text in items)
1601 if not heading:
1602 parts.append( f '<div class="criteria-group"><ol class="criteria"> { rows } </ol></div>' )
1603 continue
1604 span = items[ 0 ][ 0 ] if len (items) == 1 else f ' { items[ 0 ][ 0 ] } – { items[ - 1 ][ 0 ] } '
1605 parts.append(
1606 f '<details class="criteria-group"><summary class="criteria-title"><span> { heading } </span>'
1607 f '<span class="mono-label"> { span } · { len (items) } checks</span></summary>'
1608 f '<ol class="criteria"> { rows } </ol></details>' )
1609 return '' .join(parts)
1610
1611
1612 def render_brief (prompt, expected):
1613 """The prompt and expected outcome boxes. Both arguments are HTML.
1614 An expected outcome with one numbered or bulleted item on each line shows as numbered lists,
1615 grouped under the plain lines between them."""
1616 groups = parse_criteria(expected)
1617 if groups:
1618 total = sum ( len (items) for _, items in groups)
1619 named = sum ( 1 for heading, _ in groups if heading)
1620 label = f ' { total } checks' + ( f ' · { named } groups' if named > 1 else '' )
1621 toggle = '<button type="button" class="btn" data-expand-all>Expand all</button>' if named else ''
1622 body = render_criteria(groups)
1623 count = f '<span class="head-meta"><span class="mono-label"> { label } </span> { toggle } </span>'
1624 else :
1625 body, count = f '<div class="msg"> { code_spans(expected) } </div>' , ''
1626 words = len (html.unescape(prompt).split())
1627 prompt_meta = ( f '<span class="head-meta"><span class="mono-label">Sent to every builder · { words } words</span>'
1628 '<button type="button" class="btn" data-copy="brief-prompt">Copy</button></span>' )
1629 return (
1630 '<section class="brief folds">'
1631 + fold( 'Prompt' , prompt_meta, f '<blockquote class="msg" id="brief-prompt"> { prompt } </blockquote>' ,
1632 open_ = True , cls = 'brief-card prompt' )
1633 + fold( 'Expected outcome' , count, body, cls = 'brief-card expected' )
1634 + '</section>'
1635 )
1636
1637
1638 THEAD = ( '<tr><th>Run</th><th>Model</th><th>Effort</th><th>Result</th><th>Active time</th><th>LLM calls</th><th>Tool calls</th>'
1639 '<th>Input tokens</th><th>Cached</th><th>Output tokens</th><th>Cost</th></tr>' )
1640
1641
1642 def fill_template (title, rows, notes, panels, replay = '' , head = '' , scripts = '' , case = None , thead = THEAD ,
1643 prompt = '' , expected = '' , after_table = '' , kicker = 'Builder comparison' , meta = '' ):
1644 """The full page. Text arguments are HTML. case defaults to the prompt and expected outcome boxes.
1645 kicker is the label next to the logo, and meta the line under the title.
1646 replay comes before the table, and after_table between the table and the panels."""
1647 return ( TEMPLATE
1648 .replace( ' {{ THEAD }} ' , thead)
1649 .replace( ' {{ HEAD }} ' , head)
1650 .replace( ' {{ REPLAY }} ' , replay)
1651 .replace( ' {{ AFTER_TABLE }} ' , after_table)
1652 .replace( ' {{ SCRIPTS }} ' , scripts)
1653 .replace( ' {{ KICKER }} ' , kicker)
1654 .replace( ' {{ META }} ' , f '<p class="page-meta mono-label"> { meta } </p>' if meta else '' )
1655 .replace( ' {{ TITLE }} ' , title)
1656 .replace( ' {{ ROWS }} ' , rows)
1657 .replace( ' {{ NOTES }} ' , f '<p class="fineprint"> { notes } </p>' if notes else '' )
1658 .replace( ' {{ CASE }} ' , render_brief(prompt, expected) if case is None else case)
1659 .replace( ' {{ PANELS }} ' , panels))
1660
1661
1662 TEMPLATE = """<!doctype html>
1663 <html lang="en"><head><meta charset="utf-8">
1664 <meta name="viewport" content="width=device-width, initial-scale=1">
1665 <title> {{ TITLE }} </title>
1666 <style>
1667 @font-face{font-family:"n8n Baumer Uniwidth";src:url("fonts/n8nBaumerUniwidth.woff2") format("woff2");font-weight:400 500;font-display:swap}
1668 @font-face{font-family:"n8n Baumer Mono";src:url("fonts/n8nBaumerMono.woff2") format("woff2");font-weight:400;font-display:swap}
1669 /* n8n brand, Light mode: https://guidelines.n8n.design/ */
1670 :root{
1671 --bg:#FAFAFA;--bg-2:#F1F1F1;--bg-3:#EAEAEA;--white:#FFFFFF;--black:#191919;
1672 --line:#E0E0E0;--line-strong:#BFBFBF;--text:#191919;--dim:#676767;
1673 --pink:#FF91AC;--pink-10:#FFE8EC;--pink-20:#FFD0D9;--pink-30:#FFBCCA;--pink-70:#87003C;--gray-50:#A4A4A4;
1674 --s4:4px;--s8:8px;--s12:12px;--s16:16px;--s24:24px;--s32:32px;--s40:40px;
1675 --r4:4px;--r8:8px;--r12:12px;--full:400px;
1676 --sans:"n8n Baumer Uniwidth",system-ui,sans-serif;--mono:"n8n Baumer Mono",ui-monospace,monospace;
1677 }
1678 *{box-sizing:border-box}
1679 body{margin:0 auto;max-width:1360px;background:var(--bg);color:var(--text);font:14px/18px var(--sans);font-feature-settings:"liga" 1;padding:var(--s24) var(--s40) var(--s40)}
1680 body.theater-open{overflow:hidden}
1681 h1,h2,h3{font-weight:500;margin:0;text-wrap:balance}
1682 h1{font-size:24px;line-height:24px}
1683 h2{font-size:18px;line-height:22px}
1684 h3{font-size:16px;line-height:20px}
1685 a{color:var(--text);text-decoration:underline;text-decoration-color:var(--pink);text-decoration-thickness:2px;text-underline-offset:3px}
1686 a:hover{text-decoration-color:var(--text)}
1687 code,pre,.mono,.mono-label{font-family:var(--mono)}
1688 .mono-label{font-size:10px;line-height:14px;text-transform:uppercase;letter-spacing:.02em;color:var(--dim)}
1689 .dim{color:var(--dim)}
1690 .err{color:var(--pink-70)}
1691 button:focus-visible,select:focus-visible{outline:2px solid var(--pink);outline-offset:2px}
1692 .page-head{margin-bottom:var(--s24)}
1693 .brand-row{display:flex;align-items:center;gap:var(--s12)}
1694 .brand-row img{height:20px}
1695 .brand-row img + .mono-label{padding-left:var(--s12);box-shadow:inset 1px 0 0 var(--line-strong)}
1696 .page-head h1{margin-top:var(--s16);font-size:32px;line-height:36px;letter-spacing:-.01em;max-width:960px}
1697 .page-meta{margin:var(--s8) 0 0}
1698 .brief{margin-bottom:var(--s32)}
1699 .brief-card.prompt{background:var(--white);box-shadow:inset 0 0 0 1px var(--line),inset 4px 0 0 var(--pink);padding-left:calc(var(--s16) + 4px)}
1700 .brief-card blockquote{margin:0;font-size:18px;line-height:26px}
1701 details.criteria-group + details.criteria-group{border-top:1px solid var(--line-strong)}
1702 .criteria-title{display:flex;align-items:baseline;gap:var(--s8);padding:var(--s12) 0;font-weight:500;cursor:pointer;list-style:none}
1703 .criteria-title::-webkit-details-marker{display:none}
1704 .criteria-title::before{content:"";flex:none;align-self:center;width:6px;height:6px;margin-right:var(--s4);border-right:1.5px solid currentColor;border-bottom:1.5px solid currentColor;transform:rotate(-45deg);transition:transform .15s}
1705 details[open] > .criteria-title::before{transform:rotate(45deg)}
1706 .criteria-title .mono-label{margin-left:auto}
1707 .criteria-title:hover{color:var(--dim)}
1708 .criteria-title:focus-visible{outline:2px solid var(--pink);outline-offset:2px;border-radius:var(--r4)}
1709 details.criteria-group > .criteria{padding-bottom:var(--s12)}
1710 details.criteria-group:first-of-type > .criteria-title{padding-top:0}
1711 @media (prefers-reduced-motion:reduce){.criteria-title::before{transition:none}}
1712 .checks .mark.num{width:auto;min-width:20px;height:20px;margin-top:-1px;padding:0 var(--s4);border-radius:var(--full)}
1713 .criteria{list-style:none;margin:0;padding:0}
1714 .criteria li{display:grid;grid-template-columns:24px minmax(0,1fr);gap:var(--s12);padding:var(--s8) 0;text-wrap:pretty}
1715 .criteria li:first-child{padding-top:0}
1716 .criteria li + li{border-top:1px solid var(--line)}
1717 .criteria .num{display:grid;place-items:center;height:24px;margin-top:-3px;border-radius:var(--full);background:var(--white);box-shadow:inset 0 0 0 1px var(--line);font:10px/14px var(--mono)}
1718 .brief code{font-size:12px;padding:0 4px;border-radius:var(--r4);background:var(--white);box-shadow:inset 0 0 0 1px var(--line);word-break:break-word}
1719 .case{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--s16);margin-bottom:var(--s24)}
1720 .case .mono-label{margin-bottom:var(--s8)}
1721 .msg{white-space:pre-wrap;word-break:break-word;text-wrap:pretty}
1722 table{width:100%;border-collapse:collapse;margin-bottom:var(--s8)}
1723 th{text-align:left;font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em;color:var(--dim);padding:var(--s8) var(--s12);font-weight:400}
1724 td{padding:var(--s12);border-top:1px solid var(--line);font-family:var(--mono);white-space:nowrap}
1725 td:first-child{font-family:var(--sans)}
1726 .row{cursor:pointer}
1727 .row:hover td{background:var(--bg-2)}
1728 .row.active td{background:var(--bg-3)}
1729 .row.active td:first-child{box-shadow:inset 4px 0 0 var(--pink)}
1730 .run-name{font-weight:500}
1731 .fineprint{color:var(--dim);font-size:10px;line-height:14px;margin:0 0 var(--s24);max-width:960px;text-wrap:pretty}
1732 .badge{display:inline-block;font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em;padding:var(--s4) var(--s12);border-radius:var(--full);white-space:nowrap}
1733 .badge.pass{background:var(--pink);color:var(--black)}
1734 .badge.partial{background:var(--bg-3);color:var(--black)}
1735 .badge.fail{background:var(--pink-70);color:var(--bg)}
1736 .badge.none{box-shadow:inset 0 0 0 1px var(--line-strong);color:var(--dim)}
1737 /* Live page */
1738 .badge.st-waiting,.badge.st-skipped{box-shadow:inset 0 0 0 1px var(--line-strong);color:var(--dim)}
1739 .badge.st-running{background:var(--pink-10);color:var(--pink-70);box-shadow:inset 0 0 0 1px var(--pink-30)}
1740 .badge.st-running::before{content:"";display:inline-block;width:6px;height:6px;margin-right:var(--s8);border-radius:50%;background:var(--pink-70);vertical-align:1px;animation:pulse 1.2s ease-in-out infinite}
1741 .badge.st-done{background:var(--bg-3);color:var(--black)}
1742 .badge.st-failed{background:var(--pink-70);color:var(--bg)}
1743 @keyframes pulse{50%{opacity:.25}}
1744 @media (prefers-reduced-motion:reduce){.badge.st-running::before{animation:none}}
1745 .live-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:var(--s16)}
1746 @media (max-width:1000px){.live-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}
1747 @media (max-width:640px){.live-grid{grid-template-columns:minmax(0,1fr)}}
1748 .live-card{display:flex;flex-direction:column;min-width:0;background:var(--white);border:1px solid var(--line);border-radius:var(--r8);overflow:hidden}
1749 .live-card:has([data-live-badge] :is(.st-failed,.st-error)){background:var(--pink-10);border-color:var(--pink-70);box-shadow:0 0 0 1px var(--pink-70)}
1750 .live-card .reel-meta{align-items:flex-start;padding:var(--s12)}
1751 .live-card .run-name{padding-top:3px;word-break:break-word}
1752 .live-card [data-live-badge]{flex:none}
1753 .live-screen{position:relative;aspect-ratio:16/10;overflow:hidden;background:var(--bg-2);border-bottom:1px solid var(--line)}
1754 .live-screen:has([data-cast]){background:var(--black)}
1755 .live-screen img{display:block;width:100%;height:100%;object-fit:cover;object-position:top center;transition:transform .15s}
1756 .live-screen img[hidden]{display:none}
1757 .live-screen .live-empty{display:grid;place-items:center;height:100%;color:var(--gray-50);font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em;padding:var(--s16);text-align:center}
1758 .live-screen [data-cast],.live-screen [data-cast] .ap-wrapper{width:100%;height:100%}
1759 .live-screen .ap-control-bar,.live-screen .ap-overlay-info,.live-zoomed .ap-control-bar,.live-zoomed .ap-overlay-info{display:none !important}
1760 .live-zoom{display:block;width:100%;height:100%;padding:0;border:0;background:none;cursor:zoom-in}
1761 .live-zoom:has(img[hidden]){display:none}
1762 .live-zoom:hover img{transform:scale(1.03)}
1763 .live-zoom:focus-visible{outline:2px solid var(--pink);outline-offset:-2px}
1764 .live-zoom.overlay{position:absolute;inset:0}
1765 @media (prefers-reduced-motion:reduce){.live-screen img{transition:none}}
1766 .live-zoomed-cast{width:100%;height:100%}
1767 .live-zoomed-cast .ap-wrapper{width:100%;height:100%}
1768 .live-theater [data-zoom-close]{margin-left:auto}
1769 .live-theater{overflow:hidden}
1770 .live-zoomed{flex:1;min-height:0;display:flex;align-items:center;justify-content:center}
1771 .live-zoomed img{display:block;max-width:100%;max-height:100%;border-radius:var(--r8);box-shadow:0 0 0 1px var(--line)}
1772 .live-zoomed .live-empty{display:grid;place-items:center;width:min(100%,960px);aspect-ratio:16/10;border-radius:var(--r8);background:var(--black);color:var(--gray-50);font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em}
1773 .live-zoomed [hidden]{display:none}
1774 .head-meta{display:flex;align-items:center;gap:var(--s12)}
1775 .verdict-body{margin-top:var(--s16)}
1776 .summary{margin:0 0 var(--s16);text-wrap:pretty}
1777 .check-group + .check-group{margin-top:var(--s16)}
1778 .check-group > .mono-label{margin-bottom:var(--s4)}
1779 .checks{list-style:none;margin:0;padding:0;display:grid;gap:var(--s4)}
1780 .checks li{display:flex;align-items:flex-start;gap:var(--s8);padding:var(--s8);border-radius:var(--r8);background:var(--white);text-wrap:pretty}
1781 .checks li.bad{background:var(--pink-10);box-shadow:inset 3px 0 0 var(--pink-70)}
1782 .checks li.ok{background:none;padding-block:var(--s4)}
1783 .checks .mark{flex:none;display:grid;place-items:center;width:16px;height:16px;margin-top:1px;border-radius:50%;font:10px/1 var(--mono)}
1784 .checks .ok .mark{background:var(--pink);color:var(--black)}
1785 .checks .bad .mark{background:var(--pink-70);color:var(--bg)}
1786 .checks .warn .mark{box-shadow:inset 0 0 0 1px var(--line-strong);color:var(--dim)}
1787 .summary code,.checks code{font-size:12px;padding:0 2px;border-radius:var(--r4);background:var(--bg-3);word-break:break-word}
1788 .checks .bad code{background:var(--pink-20)}
1789 .checks .info .mark{box-shadow:inset 0 0 0 1px var(--line-strong);color:var(--dim)}
1790 .checks li.info{background:none;padding-block:var(--s4)}
1791 .checks.goal{margin-bottom:var(--s12)}
1792 .check-body{display:grid;gap:var(--s4);min-width:0}
1793 .check-note,.check-source{font-size:12px;color:var(--dim)}
1794 .check-details{margin:0;padding-left:var(--s16);font-size:12px;color:var(--dim);display:grid;gap:2px;list-style:disc}
1795 .checks .check-details li{display:list-item;padding:0;border-radius:0;background:none;box-shadow:none}
1796 .folds{margin-bottom:var(--s24)}
1797 .folds > .box{margin-bottom:var(--s12)}
1798 details.fold > summary{list-style:none;cursor:pointer;margin-bottom:0}
1799 details.fold > summary::-webkit-details-marker{display:none}
1800 details.fold > summary h2::before{content:"▸";display:inline-block;width:1em;color:var(--dim);transition:transform .15s}
1801 details.fold[open] > summary h2::before{transform:rotate(90deg)}
1802 details.fold[open] > summary{margin-bottom:var(--s12)}
1803 .canvas-box .canvas img{max-height:480px}
1804 .auto-note{margin:0 0 var(--s12);font-size:12px}
1805 .std{margin-left:var(--s8);white-space:nowrap}
1806 .run{display:none}
1807 .run.active{display:block}
1808 .box{background:var(--bg-2);border-radius:var(--r12);padding:var(--s16);margin-bottom:var(--s24)}
1809 .box.err{background:var(--pink-10);color:var(--pink-70)}
1810 .head{display:flex;justify-content:space-between;align-items:center;gap:var(--s12);margin-bottom:var(--s12)}
1811 .result-row{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--s16);margin-bottom:var(--s24)}
1812 .result-row > .box{margin-bottom:0}
1813 .canvas{display:grid;place-items:center;background:var(--bg);border-radius:var(--r8);padding:var(--s16)}
1814 .canvas img{display:block;max-width:100%;height:auto;max-height:360px;object-fit:contain}
1815 a.canvas[data-canvas]{cursor:zoom-in}
1816 /* Full-screen canvas */
1817 #canvas-view{overflow:hidden}
1818 #canvas-badge{margin-left:auto}
1819 #canvas-n8n[hidden]{display:none}
1820 .canvas-stage{position:relative;flex:1;min-height:0;display:flex;overflow:auto;background:var(--white);border-radius:var(--r12);box-shadow:inset 0 0 0 1px var(--line);padding:var(--s24)}
1821 .canvas-stage img{display:block;margin:auto;max-width:100%;max-height:100%}
1822 .canvas-stage.zoomable img{cursor:zoom-in}
1823 .canvas-stage.zoomed img{max-width:none;max-height:none;cursor:zoom-out}
1824 .btn:disabled{opacity:.4;cursor:default;background:var(--bg-2)}
1825 .btn,.tab{font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em;color:var(--black);background:var(--bg-2);border:0;box-shadow:inset 0 0 0 1px var(--line);border-radius:var(--full);padding:var(--s8) var(--s16);cursor:pointer;white-space:nowrap}
1826 .btn:hover,.tab:hover{background:var(--white)}
1827 .btn.accent,.tab.active{background:var(--pink);box-shadow:none}
1828 .btn.accent:hover,.tab.active:hover{background:var(--pink-20)}
1829 .btn.accent:active{background:var(--pink-30)}
1830 .tabs{display:flex;flex-wrap:wrap;gap:var(--s4)}
1831 /* Replay strip */
1832 .reel{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));gap:var(--s16)}
1833 .reel-card{display:flex;flex-direction:column;gap:var(--s8);min-width:0;padding:var(--s12);background:var(--white);border-radius:var(--r12);box-shadow:inset 0 0 0 1px var(--line)}
1834 .thumb{position:relative;aspect-ratio:16/10;border-radius:var(--r8);overflow:hidden;background:var(--black);box-shadow:inset 0 0 0 1px var(--line)}
1835 .thumb-poster{position:absolute;inset:0;padding:0;border:0;background:none;cursor:pointer}
1836 .thumb-poster img,.thumb-poster video{display:block;width:100%;height:100%;object-fit:cover;object-position:top left}
1837 .thumb-player{pointer-events:none}
1838 .thumb-player .ap-overlay,.thumb-player .ap-control-bar{display:none !important}
1839 .thumb-live{position:absolute;inset:0;display:grid;place-items:center}
1840 .thumb-live > video{width:100%;height:100%;object-fit:contain}
1841 .thumb-live .ap-wrapper{width:100%;height:100%}
1842 .play-icon{position:absolute;left:50%;top:50%;width:48px;height:48px;margin:-24px 0 0 -24px;border-radius:var(--full);background:var(--pink);display:grid;place-items:center;transition:transform .15s}
1843 .play-icon svg{width:24px;height:24px;fill:var(--black);margin-left:2px}
1844 .thumb-poster:hover .play-icon{transform:scale(1.12)}
1845 .reel-meta{display:flex;align-items:center;justify-content:space-between;gap:var(--s8)}
1846 /* Full-screen player */
1847 .theater{position:fixed;inset:0;z-index:10;background:var(--bg);padding:var(--s24) var(--s40);overflow:auto;display:flex;flex-direction:column;gap:var(--s16)}
1848 .theater[hidden]{display:none}
1849 .theater-bar{display:flex;flex-wrap:wrap;align-items:center;gap:var(--s16)}
1850 .theater-bar label{display:flex;align-items:center;gap:var(--s8)}
1851 .theater-bar input{accent-color:var(--black)}
1852 .theater-controls{display:flex;flex-wrap:wrap;align-items:center;gap:var(--s16);margin-left:auto}
1853 .speed{display:flex;align-items:center;gap:var(--s4)}
1854 .speed .mono-label{margin-right:var(--s4)}
1855 .speed button{font:10px/14px var(--mono);color:var(--dim);background:none;border:0;border-radius:var(--full);padding:var(--s4) var(--s8);cursor:pointer}
1856 .speed button:hover{color:var(--text);background:var(--bg-2)}
1857 .speed button[aria-pressed="true"]{color:var(--black);background:var(--bg-3)}
1858 .replay-grid{display:grid;gap:var(--s16);width:100%;margin:0 auto;grid-template-columns:minmax(0,1fr);max-width:calc((100vh - 240px) * 1.6 + 24px)}
1859 .replay-grid.all{grid-template-columns:repeat(var(--cols),var(--cell));justify-content:center;max-width:none}
1860 .screen{margin:0;display:none;padding:var(--s12);background:var(--white);border-radius:var(--r12);box-shadow:inset 0 0 0 1px var(--line)}
1861 .screen.shown{display:flex;flex-direction:column;gap:var(--s8)}
1862 .screen:has(.badge.fail){background:var(--pink-10);box-shadow:inset 0 0 0 1px var(--pink-30)}
1863 .screen-head{display:flex;align-items:center;gap:var(--s12);min-height:22px}
1864 .screen-head .badge{margin-left:auto;padding-block:var(--s4)}
1865 .screen-frame{aspect-ratio:16/10;overflow:hidden;border-radius:var(--r8);background:var(--black)}
1866 .screen-frame video{display:block;width:100%;height:100%;object-fit:contain}
1867 .player,.player .ap-wrapper{width:100%;height:100%}
1868 .theater .fineprint{margin:0}
1869 /* LLM steps */
1870 .run-picker{margin-right:auto;max-width:320px;font:12px/16px var(--sans);color:var(--text);padding:var(--s4) var(--s12);border:0;border-radius:var(--full);background:var(--white);box-shadow:inset 0 0 0 1px var(--line);cursor:pointer}
1871 .run-picker:hover{box-shadow:inset 0 0 0 1px var(--line-strong)}
1872 .legend{display:flex;flex-wrap:wrap;gap:var(--s16);margin-bottom:var(--s12)}
1873 .legend i{display:inline-block;width:8px;height:8px;border-radius:2px;margin-right:var(--s4)}
1874 .read{background:var(--gray-50)}
1875 .write{background:var(--pink)}
1876 .fresh{background:var(--black)}
1877 .outp{background:var(--dim)}
1878 .steps-layout{display:grid;grid-template-columns:minmax(240px,320px) minmax(0,1fr);gap:var(--s16);height:min(720px,calc(100vh - 96px))}
1879 .step-list{display:flex;flex-direction:column;gap:var(--s4);overflow-y:auto;min-height:0;padding:2px}
1880 .step{display:flex;flex-direction:column;gap:var(--s8);width:100%;padding:var(--s8) var(--s12);border:0;border-radius:var(--r8);background:transparent;color:inherit;font:inherit;text-align:left;cursor:pointer}
1881 .step:hover{background:var(--bg)}
1882 .step.active{background:var(--white);box-shadow:inset 4px 0 0 var(--pink),0 0 0 1px var(--line)}
1883 .step-top{display:flex;align-items:flex-start;gap:var(--s8);width:100%}
1884 .num{flex:none;display:inline-grid;place-items:center;min-width:24px;height:24px;padding:0 var(--s4);border-radius:var(--full);background:var(--bg-3);font:10px/14px var(--mono)}
1885 .step.active .num{background:var(--pink)}
1886 .step > .step-usage,.step > .mini{padding-left:calc(24px + var(--s8))}
1887 .step-flag{display:inline-flex;align-items:center;gap:var(--s4);font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em;color:var(--pink-70)}
1888 .step-flag::before{content:"";width:6px;height:6px;border-radius:50%;background:var(--pink-70)}
1889 .step-text{flex:1;min-width:0;padding-top:3px;overflow:hidden;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;word-break:break-word}
1890 code.step-text{font-size:12px;line-height:16px}
1891 .step-top .mono-label{padding-top:5px}
1892 .step-usage{display:flex;gap:var(--s8)}
1893 .mini{display:grid;grid-template-columns:minmax(0,4fr) minmax(0,1fr);gap:var(--s4);width:100%}
1894 .bar{display:flex;height:6px;background:var(--bg-3);border-radius:var(--full);overflow:hidden}
1895 .seg{display:block;height:100%}
1896 .step-pane{min-height:0;overflow-y:auto;background:var(--white);border-radius:var(--r8);box-shadow:inset 0 0 0 1px var(--line);padding:var(--s16)}
1897 .step-detail{display:flex;flex-direction:column;gap:var(--s12)}
1898 .step-detail[hidden]{display:none}
1899 .step-head{display:flex;align-items:baseline;justify-content:space-between;gap:var(--s12)}
1900 .stats{display:flex;flex-wrap:wrap;gap:var(--s24);margin:0;padding:var(--s12) var(--s16);background:var(--bg);border-radius:var(--r8)}
1901 .stats dd{margin:var(--s4) 0 0;font-size:14px;line-height:18px}
1902 .card{display:block;background:var(--bg);border-radius:var(--r8);padding:var(--s12);box-shadow:inset 4px 0 0 var(--line-strong)}
1903 .card.text{box-shadow:inset 4px 0 0 var(--pink)}
1904 .card.user{background:var(--white);box-shadow:inset 0 0 0 1px var(--line),inset 4px 0 0 var(--black)}
1905 .card.reasoning{background:transparent;box-shadow:inset 0 0 0 1px var(--line);color:var(--dim)}
1906 .card.is-error{background:var(--pink-10);box-shadow:inset 4px 0 0 var(--pink-70)}
1907 .card-head{display:flex;align-items:center;gap:var(--s12);margin-bottom:var(--s8)}
1908 .card-head > .mono-label:last-child{margin-left:auto}
1909 summary.card-head{margin-bottom:0;cursor:pointer}
1910 .card.reasoning[open] summary{margin-bottom:var(--s8)}
1911 .card.context{background:transparent;box-shadow:inset 0 0 0 1px var(--line)}
1912 .card.context[open] > summary{margin-bottom:var(--s8)}
1913 .card.context > .msg{max-height:420px;overflow:auto;font-size:12px;line-height:16px}
1914 .card.context .card-head .dim{margin-left:auto}
1915 /* Tools browser */
1916 .tools-browser{display:flex;flex-direction:column;gap:var(--s16)}
1917 .tools-bar{display:flex;flex-wrap:wrap;align-items:center;gap:var(--s12);font-size:12px;line-height:16px}
1918 .tools-filter{font:12px/16px var(--sans);padding:var(--s8) var(--s12);border:0;border-radius:var(--full);background:var(--white);box-shadow:inset 0 0 0 1px var(--line);min-width:200px}
1919 .tools-filter:focus-visible{outline:2px solid var(--pink);outline-offset:1px}
1920 .tool-groups{display:flex;flex-direction:column;gap:var(--s12);max-height:280px;overflow:auto;padding:2px}
1921 .tool-group{display:flex;flex-direction:column;gap:var(--s4)}
1922 .tool-group[hidden],.chip[hidden]{display:none}
1923 .tool-group.empty .dim,.group-note{font-size:12px;line-height:16px}
1924 .chip.runtime:not(.active){background:var(--white);border:1px dashed var(--pink-30);box-shadow:none;padding:calc(var(--s4) - 1px) calc(var(--s12) - 1px)}
1925 .chips{display:flex;flex-wrap:wrap;gap:var(--s4)}
1926 .chip{display:inline-flex;align-items:center;gap:var(--s4);font:12px/16px var(--mono);color:var(--dim);background:var(--white);border:0;box-shadow:inset 0 0 0 1px var(--line);border-radius:var(--full);padding:var(--s4) var(--s12);cursor:pointer}
1927 .chip:hover{color:var(--text);box-shadow:inset 0 0 0 1px var(--line-strong)}
1928 .chip.used{color:var(--text);background:var(--pink-10);box-shadow:inset 0 0 0 1px var(--pink-30)}
1929 .chip.active{color:var(--black);background:var(--pink);box-shadow:none}
1930 .chip .count{font-size:10px;color:var(--dim)}
1931 .chip.active .count{color:var(--black)}
1932 .fail-dot{width:6px;height:6px;border-radius:50%;background:var(--pink-70)}
1933 .tool-detail{display:flex;flex-direction:column;gap:var(--s12);padding-top:var(--s16);border-top:1px solid var(--line)}
1934 .tool-detail[hidden]{display:none}
1935 .tool-detail-head{display:flex;flex-wrap:wrap;align-items:center;gap:var(--s12)}
1936 .tool-desc{font-size:12px;line-height:16px;color:var(--dim);max-height:200px;overflow:auto}
1937 .params{margin:0;font-size:12px;line-height:16px;background:var(--white);border-radius:var(--r8);box-shadow:inset 0 0 0 1px var(--line)}
1938 .params th{padding:var(--s8)}
1939 .params td{padding:var(--s8);vertical-align:top;white-space:normal;font-family:var(--sans)}
1940 .params td:first-child{white-space:nowrap}
1941 .params code{font-size:12px}
1942 .params .type{color:var(--dim);word-break:break-word}
1943 .params .req{margin-left:var(--s8);font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em;color:var(--pink-70)}
1944 .params .limits{margin-top:var(--s4);font:10px/14px var(--mono);color:var(--dim);word-break:break-all}
1945 .tool-name{font-size:14px;line-height:18px}
1946 .note{color:var(--dim);margin-bottom:var(--s4)}
1947 .card details summary{cursor:pointer;color:var(--dim);margin-top:var(--s8);font:10px/14px var(--mono);text-transform:uppercase;letter-spacing:.02em}
1948 .card pre{background:var(--white);border-radius:var(--r4);box-shadow:inset 0 0 0 1px var(--line);padding:var(--s12);margin:var(--s8) 0 0;overflow:auto;max-height:320px;font-size:12px;line-height:16px;white-space:pre-wrap;word-break:break-word}
1949 </style>{{HEAD}}</head><body>
1950 <header class="page-head"><div class="brand-row"><img src="brand/n8n-logo-black-accent.svg" alt="n8n" onerror="this.remove()">
1951 <span class="mono-label">{{KICKER}}</span></div><h1>{{TITLE}}</h1>{{META}}</header>
1952 {{CASE}}
1953 {{REPLAY}}
1954 <table><thead>{{THEAD}}</thead>
1955 <tbody>{{ROWS}}</tbody></table>
1956 {{NOTES}}
1957 {{AFTER_TABLE}}
1958 {{PANELS}}
1959 <div class="theater" id="canvas-view" role="dialog" aria-modal="true" aria-label="Canvas" hidden>
1960 <div class="theater-bar"><h2>Canvas</h2><div class="tabs" id="canvas-tabs"></div><span id="canvas-badge"></span>
1961 <a id="canvas-n8n" target="_blank">Open in n8n ↗</a>
1962 <button type="button" class="btn" id="canvas-zoom">Zoom in</button>
1963 <button type="button" class="btn" id="canvas-close">Close</button></div>
1964 <div class="canvas-stage" id="canvas-stage"><img id="canvas-img" alt=""></div>
1965 <p class="fineprint">Pick a run, or press the left and right arrow keys. Click the canvas to zoom in where you click. Press Esc to close.</p></div>
1966 {{SCRIPTS}}
1967 <script>
1968 const $ = (id) => document.getElementById(id);
1969 // file:// pages can block the clipboard. Then the text is selected, so Cmd+C copies it.
1970 document.addEventListener('click', async e=>{
1971 const b = e.target.closest('[data-copy]');
1972 if (!b) return;
1973 e.preventDefault();
1974 const el = $(b.dataset.copy);
1975 try { await navigator.clipboard.writeText(el.textContent); b.textContent = 'Copied'; }
1976 catch { getSelection().selectAllChildren(el); b.textContent = 'Selected'; }
1977 setTimeout(()=>{ b.textContent = 'Copy'; }, 1500);
1978 });
1979 document.addEventListener('click', e=>{
1980 const b = e.target.closest('[data-expand-all]');
1981 if (!b) return;
1982 e.preventDefault();
1983 const card = b.closest('.brief-card');
1984 const groups = card.querySelectorAll('details.criteria-group');
1985 const open = [...groups].some(d=>!d.open);
1986 card.open = true;
1987 groups.forEach(d=>{ d.open = open; });
1988 });
1989 document.addEventListener('toggle', e=>{
1990 const card = e.target.closest?.('.brief-card');
1991 const b = card?.querySelector('[data-expand-all]');
1992 if (b) b.textContent = [...card.querySelectorAll('details.criteria-group')].every(d=>d.open) ? 'Collapse all' : 'Expand all';
1993 }, true);
1994 function show(id){
1995 document.querySelectorAll('.run').forEach(p=>p.classList.toggle('active',p.id===id));
1996 document.querySelectorAll('.row').forEach(r=>r.classList.toggle('active',r.dataset.run===id));
1997 }
1998
1999 // LLM steps: one list and one detail pane for each run.
2000 function selectStep(root, step, focus){
2001 const items = [...root.querySelectorAll('.step')];
2002 const item = items.find(b=>b.dataset.step===step);
2003 if (!item) return;
2004 items.forEach(b=>b.classList.toggle('active', b===item));
2005 root.querySelectorAll('.step-detail').forEach(d=>{ d.hidden = d.dataset.step !== step; });
2006 root.querySelector('.step-pane').scrollTop = 0;
2007 item.scrollIntoView({block:'nearest'});
2008 if (focus) item.focus({preventScroll:true});
2009 }
2010 // Delegated, so the steps keep working when the live page replaces them.
2011 document.addEventListener('click', e=>{
2012 const b = e.target.closest('.steps .step');
2013 if (b) selectStep(b.closest('.steps'), b.dataset.step);
2014 });
2015 document.addEventListener('keydown', e=>{
2016 const list = e.target.closest ? e.target.closest('.step-list') : null;
2017 if (!list || (e.key !== 'ArrowDown' && e.key !== 'ArrowUp')) return;
2018 e.preventDefault();
2019 const root = list.closest('.steps');
2020 const items = [...root.querySelectorAll('.step')];
2021 const i = items.findIndex(b=>b.classList.contains('active')) + (e.key === 'ArrowDown' ? 1 : -1);
2022 if (items[i]) selectStep(root, items[i].dataset.step, true);
2023 });
2024
2025 // Run picker in the LLM steps header. The live page calls it again after it replaces a panel.
2026 function addRunPickers(){
2027 const rows = [...document.querySelectorAll('.row[data-run]')];
2028 if (rows.length < 2) return;
2029 document.querySelectorAll('.run .steps > .head').forEach(head=>{
2030 if (head.querySelector('.run-picker')) return;
2031 const run = head.closest('.run').id;
2032 const select = Object.assign(document.createElement('select'), {className: 'run-picker'});
2033 select.setAttribute('aria-label', 'Run');
2034 for (const r of rows) {
2035 const title = r.querySelector('.run-name')?.textContent || r.dataset.run;
2036 select.add(new Option(title, r.dataset.run, false, r.dataset.run === run));
2037 }
2038 head.querySelector('h2').after(select);
2039 });
2040 }
2041 // The picker is in the summary of the LLM steps box. A click on it must not open or close the box.
2042 document.addEventListener('click', e=>{
2043 if (e.target.closest && e.target.closest('summary .run-picker')) e.preventDefault();
2044 });
2045 // The new run's steps open at the same place on the screen as the old run's steps, open or closed like them.
2046 document.addEventListener('change', e=>{
2047 if (!e.target.matches || !e.target.matches('.run-picker')) return;
2048 const select = e.target, old = select.closest('.steps'), top = old.getBoundingClientRect().top;
2049 show(select.value);
2050 const next = document.getElementById(select.value)?.querySelector('.steps');
2051 if (next) next.open = old.open;
2052 if (next) window.scrollBy(0, next.getBoundingClientRect().top - top);
2053 select.value = select.closest('.run').id;
2054 next?.querySelector('.run-picker')?.focus({preventScroll: true});
2055 });
2056 addRunPickers();
2057
2058 // Tools browser: a chip selects a tool, the filter hides chips. Delegated, like the steps.
2059 document.addEventListener('click', e=>{
2060 const chip = e.target.closest('.tools-browser .chip');
2061 if (!chip) return;
2062 const root = chip.closest('.tools-browser');
2063 root.querySelectorAll('.chip').forEach(c=>c.classList.toggle('active', c === chip));
2064 root.querySelectorAll('.tool-detail').forEach(d=>{ d.hidden = d.dataset.tool !== chip.dataset.tool; });
2065 });
2066 document.addEventListener('input', e=>{
2067 if (!e.target.matches || !e.target.matches('.tools-filter')) return;
2068 const root = e.target.closest('.tools-browser');
2069 const q = e.target.value.trim().toLowerCase();
2070 root.querySelectorAll('.chip').forEach(c=>{ c.hidden = !!q && !c.dataset.search.includes(q); });
2071 root.querySelectorAll('.tool-group').forEach(g=>{ g.hidden = !!q && !g.querySelector('.chip:not([hidden])'); });
2072 });
2073
2074 // Canvas: full screen, with a tab for each run that has a canvas.
2075 // The runs come from the page on each open, so the live page works when it replaces the panels.
2076 let canvasRun = null;
2077 function canvasLinks(){
2078 const seen = new Set();
2079 return [...document.querySelectorAll('a.canvas[data-canvas]')]
2080 .filter(a=>!seen.has(a.dataset.canvas) && seen.add(a.dataset.canvas));
2081 }
2082 function updateCanvasZoom(){
2083 const img = $('canvas-img'), stage = $('canvas-stage'), btn = $('canvas-zoom');
2084 const zoomed = stage.classList.contains('zoomed');
2085 const zoomable = zoomed || (img.complete && img.naturalWidth > img.clientWidth + 1);
2086 stage.classList.toggle('zoomable', zoomable);
2087 btn.disabled = !zoomable;
2088 btn.textContent = zoomed ? 'Fit to screen' : 'Zoom in';
2089 }
2090 // x and y are the clicked point as a fraction of the image. The zoom keeps that point in the middle.
2091 function zoomCanvas(on, x = 0.5, y = 0.5){
2092 const img = $('canvas-img'), stage = $('canvas-stage');
2093 if (on && !stage.classList.contains('zoomable')) return;
2094 stage.classList.toggle('zoomed', on);
2095 if (on) {
2096 stage.scrollLeft = img.offsetLeft + x * img.offsetWidth - stage.clientWidth / 2;
2097 stage.scrollTop = img.offsetTop + y * img.offsetHeight - stage.clientHeight / 2;
2098 }
2099 updateCanvasZoom();
2100 }
2101 function showCanvas(name){
2102 const a = canvasLinks().find(x=>x.dataset.canvas === name);
2103 if (!a) return;
2104 canvasRun = name;
2105 document.querySelectorAll('#canvas-tabs .tab').forEach(t=>t.classList.toggle('active', t.dataset.canvasTab === name));
2106 const badge = document.querySelector(`.row[data-run="${name}"] .badge`);
2107 $('canvas-badge').replaceChildren(...(badge ? [badge.cloneNode(true)] : []));
2108 const link = $('canvas-n8n');
2109 link.hidden = !a.dataset.n8n;
2110 link.href = a.dataset.n8n || '#';
2111 const img = $('canvas-img');
2112 $('canvas-stage').classList.remove('zoomed');
2113 img.onload = updateCanvasZoom;
2114 img.alt = `${a.dataset.title} canvas`;
2115 img.src = a.getAttribute('href');
2116 updateCanvasZoom();
2117 if (document.getElementById(name)) show(name);
2118 }
2119 function openCanvas(name){
2120 $('canvas-tabs').replaceChildren(...canvasLinks().map(a=>{
2121 const tab = Object.assign(document.createElement('button'), {type: 'button', className: 'tab', textContent: a.dataset.title});
2122 tab.dataset.canvasTab = a.dataset.canvas;
2123 return tab;
2124 }));
2125 pauseInline();
2126 $('canvas-view').hidden = false;
2127 document.body.classList.add('theater-open');
2128 showCanvas(name);
2129 $('canvas-close').focus();
2130 }
2131 function closeCanvas(){
2132 $('canvas-view').hidden = true;
2133 document.body.classList.remove('theater-open');
2134 canvasRun = null;
2135 }
2136 document.addEventListener('click', e=>{
2137 const a = e.target.closest('a.canvas[data-canvas]');
2138 if (!a || e.button || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return;
2139 e.preventDefault();
2140 openCanvas(a.dataset.canvas);
2141 });
2142 $('canvas-tabs').addEventListener('click', e=>{
2143 const tab = e.target.closest('[data-canvas-tab]');
2144 if (tab) showCanvas(tab.dataset.canvasTab);
2145 });
2146 $('canvas-zoom').addEventListener('click', ()=>zoomCanvas(!$('canvas-stage').classList.contains('zoomed')));
2147 $('canvas-img').addEventListener('click', e=>{
2148 const r = e.target.getBoundingClientRect();
2149 zoomCanvas(!$('canvas-stage').classList.contains('zoomed'), (e.clientX - r.left) / r.width, (e.clientY - r.top) / r.height);
2150 });
2151 $('canvas-close').addEventListener('click', closeCanvas);
2152 document.addEventListener('keydown', e=>{
2153 if ($('canvas-view').hidden) return;
2154 if (e.key === 'Escape') { closeCanvas(); return; }
2155 if (e.key !== 'ArrowLeft' && e.key !== 'ArrowRight') return;
2156 e.preventDefault();
2157 const names = canvasLinks().map(a=>a.dataset.canvas);
2158 const i = names.indexOf(canvasRun) + (e.key === 'ArrowRight' ? 1 : -1);
2159 showCanvas(names[(i + names.length) % names.length]);
2160 });
2161 window.addEventListener('resize', ()=>{ if (!$('canvas-view').hidden) updateCanvasZoom(); });
2162
2163 // Replay
2164 let replayMode = null;
2165 const players = {};
2166 function npt(seconds){
2167 const s = Math.max(0, Math.ceil(seconds));
2168 return `npt:${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`;
2169 }
2170 let speed = 1;
2171 // Play all turns into Pause all, and turns back when every shown screen has ended.
2172 let ended = new Set();
2173 function setPlayingAll(on){
2174 $('play-all').textContent = on ? 'Pause all' : 'Play all';
2175 $('play-all').dataset.playing = on ? '1' : '';
2176 }
2177 function screenEnded(name){
2178 ended.add(name);
2179 if (ended.size >= document.querySelectorAll('.screen.shown').length) setPlayingAll(false);
2180 }
2181 function mountCast(s, el){
2182 if (players[s.name]) players[s.name].dispose();
2183 players[s.name] = AsciinemaPlayer.create({data: window.CASTS[s.name]}, el, {
2184 speed, idleTimeLimit: $('skip-idle').checked ? 2 : undefined, fit: 'both', poster: 'npt:0:8',
2185 });
2186 players[s.name].addEventListener('ended', ()=>screenEnded(s.name));
2187 }
2188 function disposePlayers(){
2189 for (const name of Object.keys(players)) { players[name].dispose(); delete players[name]; }
2190 }
2191 // Side by side uses the column count that gives the largest screens in the window.
2192 function layoutGrid(){
2193 const grid = $('replay-grid');
2194 if (replayMode !== 'all') { grid.style.removeProperty('--cell'); grid.style.removeProperty('--cols'); return; }
2195 const gap = 16, pad = 24, head = 30, aspect = 1.6;
2196 const width = grid.parentElement.clientWidth - 80;
2197 const height = window.innerHeight - grid.getBoundingClientRect().top - 72;
2198 let best = {cols: 1, cell: 0};
2199 for (let cols = 1; cols <= SCREENS.length; cols++) {
2200 const rows = Math.ceil(SCREENS.length / cols);
2201 const byHeight = ((height - gap * (rows - 1)) / rows - pad - head) * aspect + pad;
2202 const cell = Math.min((width - gap * (cols - 1)) / cols, byHeight);
2203 if (cell > best.cell) best = {cols, cell};
2204 }
2205 grid.style.setProperty('--cols', best.cols);
2206 grid.style.setProperty('--cell', `${Math.floor(best.cell)}px`);
2207 }
2208 function showReplay(mode){
2209 replayMode = mode;
2210 ended = new Set();
2211 setPlayingAll(false);
2212 const grid = $('replay-grid');
2213 grid.classList.toggle('all', mode === 'all');
2214 layoutGrid();
2215 document.querySelectorAll('[data-replay]').forEach(b=>b.classList.toggle('active', b.dataset.replay === mode));
2216 $('play-all').style.display = $('restart-all').style.display = mode === 'all' ? '' : 'none';
2217 for (const s of SCREENS) {
2218 const fig = grid.querySelector(`.screen[data-run="${s.name}"]`);
2219 const shown = mode === 'all' || mode === s.name;
2220 fig.classList.toggle('shown', shown);
2221 if (s.type === 'cast') {
2222 // The player measures its box when it starts, so it starts again after a layout change.
2223 if (shown) mountCast(s, fig.querySelector('.player'));
2224 else if (players[s.name]) { players[s.name].dispose(); delete players[s.name]; }
2225 } else {
2226 const video = fig.querySelector('video');
2227 video.playbackRate = speed;
2228 video.onended = ()=>screenEnded(s.name);
2229 if (!shown) video.pause();
2230 }
2231 }
2232 }
2233 // A card plays in place: the poster gives way to a player with controls.
2234 const inline = {};
2235 function playInline(name){
2236 const s = SCREENS.find(x=>x.name === name);
2237 const thumb = document.querySelector(`.thumb[data-thumb="${name}"]`);
2238 const live = document.createElement('div');
2239 live.className = 'thumb-live';
2240 thumb.replaceChildren(live);
2241 if (s.type === 'cast') {
2242 inline[name] = AsciinemaPlayer.create({data: window.CASTS[name]}, live, {fit: 'both', autoPlay: true, idleTimeLimit: 2});
2243 } else {
2244 const video = Object.assign(document.createElement('video'), {src: s.src, controls: true, muted: true, playsInline: true, autoplay: true});
2245 if (s.poster) video.poster = s.poster;
2246 live.append(video);
2247 inline[name] = video;
2248 }
2249 }
2250 function pauseInline(){
2251 Object.values(inline).forEach(p=>p.pause());
2252 }
2253 function openTheater(mode){
2254 pauseInline();
2255 $('theater').hidden = false;
2256 document.body.classList.add('theater-open');
2257 showReplay(mode);
2258 $('theater-close').focus();
2259 }
2260 function closeTheater(){
2261 disposePlayers();
2262 document.querySelectorAll('#theater video').forEach(v=>v.pause());
2263 $('theater').hidden = true;
2264 document.body.classList.remove('theater-open');
2265 replayMode = null;
2266 }
2267 if (SCREENS.length) {
2268 document.querySelectorAll('[data-open]').forEach(b=>b.addEventListener('click', ()=>openTheater(b.dataset.open)));
2269 document.querySelectorAll('[data-play]').forEach(b=>b.addEventListener('click', ()=>playInline(b.dataset.play)));
2270 document.querySelectorAll('[data-replay]').forEach(b=>b.addEventListener('click', ()=>showReplay(b.dataset.replay)));
2271 // The terminal player has no speed setting after it starts, so a new speed starts the recordings again.
2272 $('speed').addEventListener('click', e=>{
2273 const b = e.target.closest('[data-speed]');
2274 if (!b) return;
2275 speed = Number(b.dataset.speed);
2276 $('speed').querySelectorAll('[data-speed]').forEach(x=>x.setAttribute('aria-pressed', String(x === b)));
2277 showReplay(replayMode);
2278 });
2279 $('skip-idle').addEventListener('change', ()=>showReplay(replayMode));
2280 $('theater-close').addEventListener('click', closeTheater);
2281 document.addEventListener('keydown', e=>{ if (e.key === 'Escape' && !$('theater').hidden) closeTheater(); });
2282 let resizeTimer;
2283 window.addEventListener('resize', ()=>{
2284 if (replayMode !== 'all') return;
2285 clearTimeout(resizeTimer);
2286 resizeTimer = setTimeout(()=>showReplay('all'), 200);
2287 });
2288 const playAll = (on)=>{
2289 if (on) ended = new Set();
2290 Object.values(players).forEach(p=>on ? p.play() : p.pause());
2291 document.querySelectorAll('.screen.shown video').forEach(v=>on ? v.play() : v.pause());
2292 setPlayingAll(on);
2293 };
2294 $('play-all').addEventListener('click', ()=>playAll(!$('play-all').dataset.playing));
2295 $('restart-all').addEventListener('click', ()=>{
2296 showReplay(replayMode);
2297 document.querySelectorAll('.screen.shown video').forEach(v=>{ v.currentTime = 0; });
2298 playAll(true);
2299 });
2300 // The strip shows the last screen of each terminal recording.
2301 for (const s of SCREENS.filter(s=>s.type === 'cast')) {
2302 const el = document.querySelector(`.thumb-player[data-cast="${s.name}"]`);
2303 if (el) AsciinemaPlayer.create({data: window.CASTS[s.name]}, el, {fit: 'width', controls: false, poster: npt(s.end)});
2304 }
2305 }
2306 </script></body></html>"""
2307
2308 if __name__ == '__main__' :
2309 main()