-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathtask_group_002.yaml
More file actions
309 lines (309 loc) · 8.38 KB
/
Copy pathtask_group_002.yaml
File metadata and controls
309 lines (309 loc) · 8.38 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
task_group_id: task_group_002
scenario_id: SCN_002_crm_b2b_quote_account_response
model: gpt-5.5, xhigh
harness: codex
conditions:
base:
overall_acc_at_3: 0.4162
overall_std_at_3: 0.0157
efficiency:
cached_input_tokens_avg_3: 281301.334
input_tokens_avg_3: 329825.4
output_tokens_avg_3: 7731.134
seconds_avg_3: 176.821
rounds_avg_3: 11.5333
tool_calls_avg_3: 30.9333
tasks:
test_001:
scores:
- 0.545455
- 0.545455
- 0.545455
acc_at_3: 0.5455
std_at_3: 0.0
cached_input_tokens_avg_3: 265600.0
input_tokens_avg_3: 306273.67
output_tokens_avg_3: 7458.33
seconds_avg_3: 163.04
rounds_avg_3: 12.0
tool_calls_avg_3: 28.0
test_002:
scores:
- 0.388889
- 0.555556
- 0.555556
acc_at_3: 0.5
std_at_3: 0.0786
cached_input_tokens_avg_3: 289834.67
input_tokens_avg_3: 344826.0
output_tokens_avg_3: 7310.67
seconds_avg_3: 168.832
rounds_avg_3: 11.0
tool_calls_avg_3: 30.6667
test_003:
scores:
- 0.25
- 0.25
- 0.25
acc_at_3: 0.25
std_at_3: 0.0
cached_input_tokens_avg_3: 281856.0
input_tokens_avg_3: 331096.33
output_tokens_avg_3: 6912.33
seconds_avg_3: 160.624
rounds_avg_3: 11.0
tool_calls_avg_3: 38.0
test_004:
scores:
- 0.30303
- 0.30303
- 0.30303
acc_at_3: 0.303
std_at_3: 0.0
cached_input_tokens_avg_3: 337962.67
input_tokens_avg_3: 389894.33
output_tokens_avg_3: 10066.67
seconds_avg_3: 231.244
rounds_avg_3: 12.6667
tool_calls_avg_3: 31.3333
test_005:
scores:
- 0.482759
- 0.482759
- 0.482759
acc_at_3: 0.4828
std_at_3: 0.0
cached_input_tokens_avg_3: 231253.33
input_tokens_avg_3: 277036.67
output_tokens_avg_3: 6907.67
seconds_avg_3: 160.363
rounds_avg_3: 11.0
tool_calls_avg_3: 26.6667
fewshot:
skill_dirs:
attempt_01: skills/task_group_002/demonstration_skill/demonstration_skill_attempt_01
attempt_02: skills/task_group_002/demonstration_skill/demonstration_skill_attempt_02
attempt_03: skills/task_group_002/demonstration_skill/demonstration_skill_attempt_03
overall_acc_at_3: 0.5117
overall_std_at_3: 0.0122
efficiency:
cached_input_tokens_avg_3: 190873.602
input_tokens_avg_3: 225481.398
output_tokens_avg_3: 7093.998
seconds_avg_3: 156.808
rounds_avg_3: 8.7333
tool_calls_avg_3: 18.2667
tasks:
test_001:
scores:
- 0.545455
- 0.545455
- 0.545455
acc_at_3: 0.5455
std_at_3: 0.0
cached_input_tokens_avg_3: 161066.67
input_tokens_avg_3: 195504.33
output_tokens_avg_3: 6527.0
seconds_avg_3: 157.497
rounds_avg_3: 8.3333
tool_calls_avg_3: 15.0
test_002:
scores:
- 0.555556
- 0.555556
- 0.555556
acc_at_3: 0.5556
std_at_3: 0.0
cached_input_tokens_avg_3: 187178.67
input_tokens_avg_3: 213154.33
output_tokens_avg_3: 7296.33
seconds_avg_3: 155.125
rounds_avg_3: 9.6667
tool_calls_avg_3: 20.0
test_003:
scores:
- 0.583333
- 0.583333
- 0.583333
acc_at_3: 0.5833
std_at_3: 0.0
cached_input_tokens_avg_3: 173440.0
input_tokens_avg_3: 201113.33
output_tokens_avg_3: 5125.33
seconds_avg_3: 114.651
rounds_avg_3: 8.3333
tool_calls_avg_3: 17.3333
test_004:
scores:
- 0.454545
- 0.393939
- 0.393939
acc_at_3: 0.4141
std_at_3: 0.0286
cached_input_tokens_avg_3: 238592.0
input_tokens_avg_3: 293782.0
output_tokens_avg_3: 10182.0
seconds_avg_3: 215.977
rounds_avg_3: 9.6667
tool_calls_avg_3: 22.0
test_005:
scores:
- 0.482759
- 0.482759
- 0.413793
acc_at_3: 0.4598
std_at_3: 0.0325
cached_input_tokens_avg_3: 194090.67
input_tokens_avg_3: 223853.0
output_tokens_avg_3: 6339.33
seconds_avg_3: 140.79
rounds_avg_3: 7.6667
tool_calls_avg_3: 17.0
self:
skill_dirs:
attempt_01: skills/task_group_002/self/self_attempt_01
attempt_02: skills/task_group_002/self/self_attempt_02
attempt_03: skills/task_group_002/self/self_attempt_03
overall_acc_at_3: 0.4536
overall_std_at_3: 0.0314
efficiency:
cached_input_tokens_avg_3: 155861.33
input_tokens_avg_3: 188075.67
output_tokens_avg_3: 6255.93
rounds_avg_3: 7.9333
tool_calls_avg_3: 17.2
tasks:
test_001:
scores:
- 0.454545
- 0.454545
- 0.454545
acc_at_3: 0.4545
std_at_3: 0.0
cached_input_tokens_avg_3: 179925.33
input_tokens_avg_3: 218686.67
output_tokens_avg_3: 6777.0
rounds_avg_3: 9.0
tool_calls_avg_3: 17.6667
test_002:
scores:
- 0.555556
- 0.555556
- 0.555556
acc_at_3: 0.5556
std_at_3: 0.0
cached_input_tokens_avg_3: 147328.0
input_tokens_avg_3: 184162.0
output_tokens_avg_3: 5849.67
rounds_avg_3: 7.6667
tool_calls_avg_3: 18.0
test_003:
scores:
- 0.25
- 0.583333
- 0.583333
acc_at_3: 0.4722
std_at_3: 0.1571
cached_input_tokens_avg_3: 144256.0
input_tokens_avg_3: 172999.0
output_tokens_avg_3: 4925.33
rounds_avg_3: 7.6667
tool_calls_avg_3: 15.6667
test_004:
scores:
- 0.30303
- 0.30303
- 0.30303
acc_at_3: 0.303
std_at_3: 0.0
cached_input_tokens_avg_3: 178133.33
input_tokens_avg_3: 203662.67
output_tokens_avg_3: 8262.67
rounds_avg_3: 8.3333
tool_calls_avg_3: 19.6667
test_005:
scores:
- 0.482759
- 0.482759
- 0.482759
acc_at_3: 0.4828
std_at_3: 0.0
cached_input_tokens_avg_3: 129664.0
input_tokens_avg_3: 160868.0
output_tokens_avg_3: 5465.0
rounds_avg_3: 7.0
tool_calls_avg_3: 15.0
reflect-3:
skill_dirs:
attempt_01: skills/task_group_002/reflect-3/reflect-3_attempt_01
attempt_02: skills/task_group_002/reflect-3/reflect-3_attempt_02
attempt_03: skills/task_group_002/reflect-3/reflect-3_attempt_03
overall_acc_at_3: 0.4424
overall_std_at_3: 0.0559
efficiency:
cached_input_tokens_avg_3: 159982.93
input_tokens_avg_3: 203435.47
output_tokens_avg_3: 7124.34
rounds_avg_3: 8.0667
tool_calls_avg_3: 16.2
tasks:
test_001:
scores:
- 0.454545
- 0.454545
- 0.545455
acc_at_3: 0.4848
std_at_3: 0.0429
cached_input_tokens_avg_3: 170709.33
input_tokens_avg_3: 223758.67
output_tokens_avg_3: 7462.67
rounds_avg_3: 8.0
tool_calls_avg_3: 10.6667
test_002:
scores:
- 0.555556
- 0.555556
- 0.555556
acc_at_3: 0.5556
std_at_3: 0.0
cached_input_tokens_avg_3: 151509.33
input_tokens_avg_3: 189616.33
output_tokens_avg_3: 7252.67
rounds_avg_3: 8.0
tool_calls_avg_3: 13.6667
test_003:
scores:
- 0.25
- 0.25
- 0.583333
acc_at_3: 0.3611
std_at_3: 0.1571
cached_input_tokens_avg_3: 143402.67
input_tokens_avg_3: 182796.33
output_tokens_avg_3: 5740.0
rounds_avg_3: 7.3333
tool_calls_avg_3: 17.0
test_004:
scores:
- 0.393939
- 0.363636
- 0.363636
acc_at_3: 0.3737
std_at_3: 0.0143
cached_input_tokens_avg_3: 173269.33
input_tokens_avg_3: 211471.67
output_tokens_avg_3: 8251.67
rounds_avg_3: 9.0
tool_calls_avg_3: 18.0
test_005:
scores:
- 0.482759
- 0.344828
- 0.482759
acc_at_3: 0.4368
std_at_3: 0.065
cached_input_tokens_avg_3: 161024.0
input_tokens_avg_3: 209534.33
output_tokens_avg_3: 6914.67
rounds_avg_3: 8.0
tool_calls_avg_3: 21.6667