Setting the file. One moment.
Merge Verdicts · N8n:testmax N8n AI · n8n-io/n8n · Skills Docs
Back to the top of the page scripts/ merge_verdicts.py
Python · 122 lines · 5 KB
16
17 import json
18 import sys
19 from collections import Counter
20 from pathlib import Path
21
22 sys.path.insert( 0 , str (Path( __file__ ).resolve().parent))
23 from build_report import load_case, parse_criteria # noqa: E402
24
25 VERSION = 2
26 VOTE_TEXT = { True : 'met' , False : 'not met' , None : 'not confirmed' }
27
28
29 def majority (values):
30 """The value that more than half of the votes have, or None when no value has a majority."""
31 value, count = Counter(values).most_common( 1 )[ 0 ]
32 return value if count * 2 > len (values) else None
33
34
35 def first_check (judge, cid):
36 return next ((c for c in judge.get( 'checks' ) or [] if str (c.get( 'id' )) == cid), None )
37
38
39 def merge_quality (judges):
40 """The quality checks by majority, in the order of the first judge that answered each check."""
41 ids = list ( dict .fromkeys( str (q.get( 'id' )) for j in judges for q in j.get( 'quality' ) or [] if q.get( 'id' )))
42 merged = []
43 for qid in ids:
44 found = [ next ((q for q in j.get( 'quality' ) or [] if str (q.get( 'id' )) == qid), None ) for j in judges]
45 votes = [q.get( 'ok' ) if q else None for q in found]
46 ok = majority(votes)
47 pick = next ((q for q, v in zip (found, votes) if q and v == ok), None ) or next (q for q in found if q)
48 item = { 'id' : qid, 'ok' : ok, 'text' : pick.get( 'text' , '' ), 'votes' : votes}
49 if len ( set (votes)) > 1 :
50 item[ 'note' ] = f ' { Counter(votes)[ok] } of { len (judges) } judges.' if ok is not None else 'Judges disagree.'
51 merged.append(item)
52 return merged
53
54
55 def merge_builder (judges, criteria_ids):
56 n = len (judges)
57 lead = judges[ 0 ]
58 checks, disagreements = [], []
59 for cid in criteria_ids:
60 found = [first_check(j, cid) for j in judges]
61 votes = [c.get( 'ok' ) if c else None for c in found]
62 if all (c is None for c in found):
63 continue
64 ok = majority(votes)
65 pick = next ((c for c, v in zip (found, votes) if c and v == ok), None ) or next (c for c in found if c)
66 check = {k: v for k, v in pick.items() if k != 'ok' }
67 check.update( id = cid, ok = ok, votes = votes)
68 if len ( set (votes)) > 1 :
69 counts = ', ' .join( f ' { count } { VOTE_TEXT [v] } ' for v, count in Counter(votes).items())
70 disagreements.append({ 'id' : cid, 'votes' : votes, 'texts' : [c.get( 'text' ) if c else None for c in found]})
71 note = f 'Judges disagree ( { counts } ).' if ok is None else f ' { Counter(votes)[ok] } of { n } judges.'
72 check[ 'note' ] = note
73 checks.append(check)
74 checks += [ dict (c) for c in lead.get( 'checks' ) or [] if str (c.get( 'id' )) not in criteria_ids]
75
76 goal_votes = [j.get( 'goalMet' ) for j in judges]
77 goal = majority(goal_votes)
78 if goal is None :
79 goal = False if goal_votes.count( False ) * 2 >= n else None
80 if len ( set (goal_votes)) > 1 :
81 reasons = [j.get( 'goalReason' ) for j in judges if j.get( 'goalMet' ) is False and j.get( 'goalReason' )]
82 disagreements.append({ 'id' : 'goal' , 'votes' : goal_votes, 'texts' : reasons})
83 verdicts = [j.get( 'verdict' ) for j in judges if j.get( 'verdict' )]
84 return {
85 'quality' : merge_quality(judges),
86 'version' : VERSION ,
87 'judges' : n,
88 'verdict' : majority(verdicts) if verdicts else None ,
89 'summary' : lead.get( 'summary' , '' ),
90 'goalMet' : goal,
91 'goalReason' : next ((j.get( 'goalReason' ) for j in judges if j.get( 'goalMet' ) is goal and j.get( 'goalReason' )), '' ),
92 'checks' : checks,
93 'disagreements' : disagreements,
94 }
95
96
97 def main ():
98 run_dir = Path(sys.argv[ 1 ] if len (sys.argv) > 1 else '.' )
99 judge_dir = run_dir / 'judge'
100 blind_map = json.loads((judge_dir / 'blind-map.json' ).read_text())
101 files = sorted (judge_dir.glob( 'verdicts-*.json' ))
102 if not files:
103 sys.exit( f 'No judge/verdicts-*.json in { run_dir } .' )
104 panel = [json.loads(f.read_text()) for f in files]
105 groups = parse_criteria(load_case(run_dir).get( 'expected' ) or '' ) or []
106 criteria_ids = [num for _, items in groups for num, _ in items]
107
108 merged = {}
109 for letter, builder in blind_map.items():
110 judges = [p[letter] for p in panel if letter in p]
111 if not judges:
112 print ( f ' { builder } : no judge gave a verdict for builder { letter } ' )
113 continue
114 merged[builder] = merge_builder(judges, criteria_ids)
115 split = len (merged[builder][ 'disagreements' ])
116 print ( f ' { builder } (builder { letter } ): { len (judges) } judges, { split } disagreement { "" if split == 1 else "s" } ' )
117 (run_dir / 'verdicts.json' ).write_text(json.dumps(merged, indent = 2 , ensure_ascii = False ))
118 print (run_dir / 'verdicts.json' )
119
120
121 if __name__ == '__main__' :
122 main()