Setting the file. One moment. Raw file
evals/baseline.json
JSON·151 lines·4 KB
9 "results": [
10 {
11 "query": "create the jira tickets from this tasks.md",
12 "should_trigger": true,
13 "triggers": 3,
14 "runs": 3,
15 "trigger_rate": 1.0
16 },
17 {
18 "query": "turn the breakdown's task decomposition into Jira tickets",
19 "should_trigger": true,
20 "triggers": 3,
21 "runs": 3,
22 "trigger_rate": 1.0
23 },
24 {
25 "query": "file the epic and its child stories for this breakdown in Jira",
26 "should_trigger": true,
27 "triggers": 3,
28 "runs": 3,
29 "trigger_rate": 1.0
30 },
31 {
32 "query": "the tasks.md is done \u2014 make tickets out of each task under the epic",
33 "should_trigger": true,
34 "triggers": 2,
35 "runs": 3,
36 "trigger_rate": 0.6666666666666666
37 },
38 {
39 "query": "I need these breakdown tasks filed as stories in Jira, can you set that up",
40 "should_trigger": true,
41 "triggers": 3,
42 "runs": 3,
43 "trigger_rate": 1.0
44 },
45 {
46 "query": "take tasks.md and create the child issues with their acceptance criteria",
47 "should_trigger": true,
48 "triggers": 3,
49 "runs": 3,
50 "trigger_rate": 1.0
51 },
52 {
53 "query": "put the breakdown work into Jira as an epic plus stories and wire the blocked-by links",
54 "should_trigger": true,
55 "triggers": 3,
56 "runs": 3,
57 "trigger_rate": 1.0
58 },
59 {
60 "query": "draft and create the Jira tickets for the tasks we just decomposed",
61 "should_trigger": true,
62 "triggers": 3,
63 "runs": 3,
64 "trigger_rate": 1.0
65 },
66 {
67 "query": "the breakdown's approved \u2014 let's get its tasks.md tasks into Jira",
68 "should_trigger": true,
69 "triggers": 3,
70 "runs": 3,
71 "trigger_rate": 1.0
72 },
73 {
74 "query": "create a story per task in tasks.md and link the dependencies",
75 "should_trigger": true,
76 "triggers": 3,
77 "runs": 3,
78 "trigger_rate": 1.0
79 },
80 {
81 "query": "what's PM-XXXX about? give me context on it",
82 "should_trigger": false,
83 "triggers": 0,
84 "runs": 3,
85 "trigger_rate": 0.0
86 },
87 {
88 "query": "deep dive PM-YYYY and its blocked dependencies",
89 "should_trigger": false,
90 "triggers": 0,
91 "runs": 3,
92 "trigger_rate": 0.0
93 },
94 {
95 "query": "update the description on PM-ZZZZ, it's already created",
96 "should_trigger": false,
97 "triggers": 0,
98 "runs": 3,
99 "trigger_rate": 0.0
100 },
101 {
102 "query": "change the acceptance criteria on this existing ticket",
103 "should_trigger": false,
104 "triggers": 0,
105 "runs": 3,
106 "trigger_rate": 0.0
107 },
108 {
109 "query": "decompose this breakdown plan into a tasks.md",
110 "should_trigger": false,
111 "triggers": 0,
112 "runs": 3,
113 "trigger_rate": 0.0
114 },
115 {
116 "query": "how do Jira epics and stories relate to each other?",
117 "should_trigger": false,
118 "triggers": 0,
119 "runs": 3,
120 "trigger_rate": 0.0
121 },
122 {
123 "query": "search Jira for open bugs in the exporter component",
124 "should_trigger": false,
125 "triggers": 0,
126 "runs": 3,
127 "trigger_rate": 0.0
128 },
129 {
130 "query": "move the finished breakdown into its team's complete folder",
131 "should_trigger": false,
132 "triggers": 0,
133 "runs": 3,
134 "trigger_rate": 0.0
135 },
136 {
137 "query": "file a bug for the crash I just fixed",
138 "should_trigger": false,
139 "triggers": 0,
140 "runs": 3,
141 "trigger_rate": 0.0
142 },
143 {
144 "query": "create a story for this work and link it to PM-XXXX",
145 "should_trigger": false,
146 "triggers": 0,
147 "runs": 3,
148 "trigger_rate": 0.0
149 }
150 ]
151}