Setting the file. One moment.
Run Real Eval · Creating Pull Request · bitwarden/ai-plugins · Skills Docs
Back to the top of the page 16
Force Multiplier
evals/run_real_eval.py
evals/ run_real_eval.py
Python · 187 lines · 8 KB
15 installed plugins.
16 """
17
18 import argparse
19 import json
20 import os
21 import select
22 import subprocess
23 import sys
24 import time
25 from concurrent.futures import ProcessPoolExecutor, as_completed
26 from pathlib import Path
27
28 TARGET_SKILL_TOKEN = "creating-pull-request"
29
30
31 def run_query (query: str , timeout: int , model: str ) -> dict :
32 cmd = [
33 "claude" ,
34 "-p" , query,
35 "--output-format" , "stream-json" ,
36 "--verbose" ,
37 "--include-partial-messages" ,
38 "--model" , model,
39 ]
40 env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE" }
41 process = subprocess.Popen(
42 cmd,
43 stdout = subprocess. PIPE ,
44 stderr = subprocess. DEVNULL ,
45 env = env,
46 )
47
48 triggered = False
49 first_skill_seen = None
50 start = time.time()
51 buffer = ""
52 pending = None
53 accum = ""
54
55 try :
56 while time.time() - start < timeout:
57 if process.poll() is not None :
58 rest = process.stdout.read()
59 if rest:
60 buffer += rest.decode( "utf-8" , errors = "replace" )
61 break
62 ready, _, _ = select.select([process.stdout], [], [], 1.0 )
63 if not ready:
64 continue
65 chunk = os.read(process.stdout.fileno(), 8192 )
66 if not chunk:
67 break
68 buffer += chunk.decode( "utf-8" , errors = "replace" )
69
70 while " \n " in buffer:
71 line, buffer = buffer.split( " \n " , 1 )
72 line = line.strip()
73 if not line:
74 continue
75 try :
76 event = json.loads(line)
77 except json.JSONDecodeError:
78 continue
79
80 if event.get( "type" ) == "stream_event" :
81 se = event.get( "event" , {})
82 if se.get( "type" ) == "content_block_start" :
83 cb = se.get( "content_block" , {})
84 if cb.get( "type" ) == "tool_use" and cb.get( "name" ) in ( "Skill" , "Read" ):
85 pending = cb.get( "name" )
86 accum = ""
87 # Other tool types are ignored — we only care whether the
88 # target skill is invoked at some point in the response.
89 elif se.get( "type" ) == "content_block_delta" and pending:
90 delta = se.get( "delta" , {})
91 if delta.get( "type" ) == "input_json_delta" :
92 accum += delta.get( "partial_json" , "" )
93 if TARGET_SKILL_TOKEN in accum:
94 return { "triggered" : True , "first_skill" : accum}
95 elif se.get( "type" ) == "content_block_stop" and pending:
96 if first_skill_seen is None :
97 first_skill_seen = accum
98 # Keep scanning past unrelated Skill/Read invocations so
99 # the eval is portable across accounts that auto-fire
100 # session-init or workflow skills before the task skill.
101 pending = None
102 accum = ""
103 elif event.get( "type" ) == "assistant" :
104 msg = event.get( "message" , {})
105 for item in msg.get( "content" , []):
106 if item.get( "type" ) != "tool_use" :
107 continue
108 name = item.get( "name" )
109 inp = item.get( "input" , {})
110 if name == "Skill" and TARGET_SKILL_TOKEN in inp.get( "skill" , "" ):
111 return { "triggered" : True , "first_skill" : inp.get( "skill" )}
112 if name == "Read" and TARGET_SKILL_TOKEN in inp.get( "file_path" , "" ):
113 return { "triggered" : True , "first_skill" : inp.get( "file_path" )}
114 elif event.get( "type" ) == "result" :
115 return { "triggered" : triggered, "first_skill" : first_skill_seen}
116 finally :
117 if process.poll() is None :
118 process.kill()
119 process.wait()
120 return { "triggered" : triggered, "first_skill" : first_skill_seen}
121
122
123 def runs_for (query, should_trigger, runs, timeout, model):
124 triggers = 0
125 samples = []
126 for _ in range (runs):
127 r = run_query(query, timeout, model)
128 if r[ "triggered" ]:
129 triggers += 1
130 samples.append(r.get( "first_skill" ))
131 rate = triggers / runs
132 # Surface samples to stderr only when the per-query outcome disagrees with
133 # `should_trigger`, so debugging info is available without baking
134 # environment-specific tool inputs (absolute paths, etc.) into the
135 # persisted result that the README diffs for regression checks.
136 if (rate >= 0.5 ) != should_trigger:
137 for s in samples:
138 print ( f " sample: { s } " , file = sys.stderr)
139 return {
140 "query" : query,
141 "should_trigger" : should_trigger,
142 "triggers" : triggers,
143 "runs" : runs,
144 "trigger_rate" : rate,
145 }
146
147
148 def main ():
149 parser = argparse.ArgumentParser()
150 parser.add_argument( "--eval-set" , required = True )
151 parser.add_argument( "--runs-per-query" , type = int , default = 3 )
152 parser.add_argument( "--num-workers" , type = int , default = 8 )
153 parser.add_argument( "--timeout" , type = int , default = 60 )
154 parser.add_argument( "--model" , default = "claude-opus-4-7" )
155 args = parser.parse_args()
156
157 eval_set = json.loads(Path(args.eval_set).read_text())
158 results = [ None ] * len (eval_set)
159 with ProcessPoolExecutor( max_workers = args.num_workers) as pool:
160 futures = {
161 pool.submit(runs_for, e[ "query" ], e[ "should_trigger" ], args.runs_per_query, args.timeout, args.model): i
162 for i, e in enumerate (eval_set)
163 }
164 for fut in as_completed(futures):
165 i = futures[fut]
166 results[i] = fut.result()
167 r = results[i]
168 tag = "PASS" if (r[ "trigger_rate" ] >= 0.5 ) == r[ "should_trigger" ] else "FAIL"
169 print ( f " [ { tag } ] rate= { r[ 'triggers' ] } / { r[ 'runs' ] } expected= { r[ 'should_trigger' ] } : { r[ 'query' ][: 80 ] } " , file = sys.stderr)
170
171 triggers_pass = sum ( 1 for r in results if r[ "should_trigger" ] and r[ "trigger_rate" ] >= 0.5 )
172 triggers_total = sum ( 1 for r in results if r[ "should_trigger" ])
173 no_trigger_pass = sum ( 1 for r in results if not r[ "should_trigger" ] and r[ "trigger_rate" ] < 0.5 )
174 no_trigger_total = sum ( 1 for r in results if not r[ "should_trigger" ])
175
176 summary = {
177 "should_trigger_pass_rate" : triggers_pass / triggers_total if triggers_total else None ,
178 "should_not_trigger_pass_rate" : no_trigger_pass / no_trigger_total if no_trigger_total else None ,
179 "should_trigger_pass" : f " { triggers_pass } / { triggers_total } " ,
180 "should_not_trigger_pass" : f " { no_trigger_pass } / { no_trigger_total } " ,
181 "results" : results,
182 }
183 print (json.dumps(summary, indent = 2 , default = str ))
184
185
186 if __name__ == "__main__" :
187 main()