{ "meta": { "created_at": "2026-10-16T12:07:55.916585Z", "updated_at": "2026-10-16T13:59:07.625391Z", "git_sha": "e288982", "harness": "minimal", "simulated": false, "models": { "nano": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "super": "nvidia/nemotron-3-super-120b-a12b", "ultra": "nvidia/Nemotron-3-Ultra-550b-a55b" }, "agent_tier": "super", "labels_path": "bench/labels/labels.json", "n_labels": 35 }, "runs": { "noise": { "config": { "name": "noise", "description": "Unchanged skills replayed 3x (pair-level pass rate = noise floor).", "experiments": [ "E6" ], "mode": "noise", "cascade": { "mode": "cascade", "level0": true, "drift": true, "controls": true }, "single_judge_blocks": null, "types": [ "R1", "R2", "R3", "R4", "R5", "R6", "N0", "D1" ], "noise_repeats": 3, "runner": "local", "concurrency": 2, "reuse_replays": null, "max_traces": 40 }, "config_path": "bench/configs/noise.yaml", "command": "ratchet bench run --config bench/configs/noise.yaml --out bench/results/2026-10-16 --limit 8 --yes", "started_at": "2026-10-16T12:07:55.928401Z", "updated_at": "2026-10-16T12:14:41.171343Z", "edits": [ { "bench_id": "noise-csv-cleaner-1", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_b1f1a81b552a", "verdict_id": "verdict_b1f1a81b552a", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: csv-cleaner still handles all 13 past tasks.", "narrative": "Passed: csv-cleaner still handles all 13 past tasks. The replay confirmed zero regressions, drift, or uncertain outcomes across every historical pair. Cost was negligible at under two cents even on the largest model. Nothing to do.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_2edad2cb09a2", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1225, "completion_tokens": 207, "usd": 0.00012318 }, { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2022, "completion_tokens": 291, "usd": 0.0008685 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_197a6f322c89", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 132, "usd": 0.00010602 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_67a9d065679f", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 214, "usd": 0.00012642 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2665, "completion_tokens": 272, "usd": 0.0010443 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_e38ae9922c61", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 162, "usd": 0.0001152 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_32816a51b570", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 94, "usd": 0.00009672 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2434, "completion_tokens": 308, "usd": 0.0010074 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2392, "completion_tokens": 1641, "usd": 0.007315 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_2825fb091a9c", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 169, "usd": 0.0001179 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_5d6ba6f554ed", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1268, "completion_tokens": 190, "usd": 0.00012168 }, { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2539, "completion_tokens": 267, "usd": 0.001002 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_f5ffc9524371", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 200, "usd": 0.00012222 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2386, "completion_tokens": 333, "usd": 0.0010155 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2338, "completion_tokens": 3271, "usd": 0.012151 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_58c1a62b9e89", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 110, "usd": 0.00010266 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_a82bab47354b", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 165, "usd": 0.00011358 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_a9098c596a66", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1114, "completion_tokens": 91, "usd": 0.00008868 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e0d0311f2d9e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 226, "usd": 0.000615 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1258, "completion_tokens": 1130, "usd": 0.004648 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_0d64fa25f9c1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 82, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00141192, "super": 0.0055527, "ultra": 0.024114 }, "counterfactual_ultra_usd": 0.064267, "spent_usd_by_tier": { "nano": 0.00174882, "super": 0.0055527, "ultra": 0.024513 }, "prompt_tokens_by_tier": { "nano": 19623, "super": 256756, "ultra": 6234 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 106.986, "simulated_models": false, "repeat": 1, "error": null }, { "bench_id": "noise-expense-report-1", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_3caa7b4656d1", "verdict_id": "verdict_3caa7b4656d1", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The replay confirmed zero regressions, zero drifts, and zero uncertain outcomes across the full evaluation set. Cost was negligible at roughly three-tenths of a cent for the nano model. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_66280b612073", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1094, "completion_tokens": 115, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_22070319fc2f", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1437, "completion_tokens": 125, "usd": 0.00011622 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3158, "completion_tokens": 263, "usd": 0.0011841 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_ea3fb3dff29f", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.96, "prompt_tokens": 1640, "completion_tokens": 170, "usd": 0.0001392 }, { "tier": "super", "verdict": "regressed", "confidence": 0.96, "prompt_tokens": 3611, "completion_tokens": 414, "usd": 0.0014559 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3606, "completion_tokens": 1212, "usd": 0.007242 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_c48b55bfccc0", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1289, "completion_tokens": 146, "usd": 0.00011238 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_853c701d2d77", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1453, "completion_tokens": 193, "usd": 0.0001335 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_d42d6dce83d6", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1466, "completion_tokens": 114, "usd": 0.00011532 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_4686e59859bd", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1291, "completion_tokens": 146, "usd": 0.0001125 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_d56754ea2bdb", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1320, "completion_tokens": 133, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_415b9fdec05e", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1497, "completion_tokens": 129, "usd": 0.00012078 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_4dc562745773", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1447, "completion_tokens": 196, "usd": 0.00013386 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_e1f47f275c76", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 168, "usd": 0.0001143 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_b74fe913e0b4", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1268, "completion_tokens": 236, "usd": 0.00013272 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2550, "completion_tokens": 268, "usd": 0.0010062 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_31fd385eebc9", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 162, "usd": 0.00011286 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_39ba0b5153d3", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 167, "usd": 0.00011424 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2300, "completion_tokens": 274, "usd": 0.0009366 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_b9aa4e37a483", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 218, "usd": 0.00012864 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_a22aec90fcb5", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1248, "completion_tokens": 98, "usd": 0.0000984 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2539, "completion_tokens": 303, "usd": 0.0010344 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2387, "completion_tokens": 2247, "usd": 0.009128 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_b028d4f82032", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1225, "completion_tokens": 215, "usd": 0.0001251 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_74a4261e6044", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 197, "usd": 0.00012462 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_8df8f80f269c", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 105, "usd": 0.00009942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2384, "completion_tokens": 301, "usd": 0.0009861 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2296, "completion_tokens": 2142, "usd": 0.008722 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e0d6d407ab13", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 133, "usd": 0.0001002 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1344, "completion_tokens": 201, "usd": 0.0005841 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_6e2bdd5f4311", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 73, "usd": 0.000087 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 218, "usd": 0.0006078 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1234, "completion_tokens": 2057, "usd": 0.007405 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3043b03818f7", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 82, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00251652, "super": 0.0077952, "ultra": 0.032497 }, "counterfactual_ultra_usd": 0.097102, "spent_usd_by_tier": { "nano": 0.00299418, "super": 0.0077952, "ultra": 0.032908 }, "prompt_tokens_by_tier": { "nano": 33975, "super": 416513, "ultra": 9763 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 110.405, "simulated_models": false, "repeat": 1, "error": null }, { "bench_id": "noise-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_3b4200c5ebd5", "verdict_id": "verdict_3b4200c5ebd5", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The replay confirmed zero regressions, zero drifts, and zero uncertain outcomes across every historical pair. Costs remain negligible at fractions of a cent per tier. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_dc438f824383", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 129, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_715cddcc221c", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1340, "completion_tokens": 160, "usd": 0.0001188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_a8c397ef0737", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1328, "completion_tokens": 169, "usd": 0.00012024 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_c132be88499e", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 93, "usd": 0.00010524 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_bd9f55f09810", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 263, "usd": 0.00014526 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_aff124ae9a37", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1408, "completion_tokens": 140, "usd": 0.00011808 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_f36ae4b363d7", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 127, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ee3f129702b0", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1343, "completion_tokens": 214, "usd": 0.00013194 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_9076f7589e4f", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 124, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_5014b33377e2", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 163, "usd": 0.00012312 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_a35aab980bff", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 126, "usd": 0.00009852 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1344, "completion_tokens": 224, "usd": 0.0006048 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_646398be752a", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 74, "usd": 0.00008724 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 226, "usd": 0.000615 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1206, "completion_tokens": 1485, "usd": 0.005661 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_7d47d0a9ae6c", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 104, "usd": 0.00009618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00148662, "super": 0.0012198, "ultra": 0.005661 }, "counterfactual_ultra_usd": 0.032618, "spent_usd_by_tier": { "nano": 0.00157338, "super": 0.0012198, "ultra": 0.006074 }, "prompt_tokens_by_tier": { "nano": 18391, "super": 146896, "ultra": 1457 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 69.066, "simulated_models": false, "repeat": 1, "error": null }, { "bench_id": "noise-github-triage-1", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_d4743fcfa022", "verdict_id": "verdict_d4743fcfa022", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The ratchet check caught a deterministic side-effect regression where the edited skill made a new github.comment call that the original run never made, and the sandbox refused it. This occurred in the \"Triage acme/data-pipeline\" replay at step t005. The other 12 pairs passed with no drift or uncertain results. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_60cfdf60bb66", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 145, "usd": 0.00010458 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_c5a1e24688d7", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 145, "usd": 0.0001071 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_b00ffbd4eb0b", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 140, "usd": 0.00011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_81f14df647b9", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 177, "usd": 0.00012486 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c351fb2ef2fe", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 97, "usd": 0.00010242 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_50facdd91f74", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "pass", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 3, "level0_changed": 1 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_e4234978c730", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1410, "completion_tokens": 188, "usd": 0.00012972 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2814, "completion_tokens": 291, "usd": 0.0011061 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_baee4d285d7b", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1341, "completion_tokens": 99, "usd": 0.00010422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_1a3e00dbdbc1", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1409, "completion_tokens": 88, "usd": 0.00010566 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3263, "completion_tokens": 424, "usd": 0.0013605 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3219, "completion_tokens": 1383, "usd": 0.007368 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_74b6a503b49a", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1364, "completion_tokens": 131, "usd": 0.00011328 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_5b401fc4bf0b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1114, "completion_tokens": 98, "usd": 0.00009036 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_609315ecf4b2", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 219, "usd": 0.0006087 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 1254, "completion_tokens": 2545, "usd": 0.008889 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_75e85443922a", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 107, "usd": 0.00009696 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00128184, "super": 0.0030753, "ultra": 0.016257 }, "counterfactual_ultra_usd": 0.046385, "spent_usd_by_tier": { "nano": 0.00159468, "super": 0.0030753, "ultra": 0.016838 }, "prompt_tokens_by_tier": { "nano": 19346, "super": 244989, "ultra": 4769 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 103.086, "simulated_models": false, "repeat": 1, "error": null }, { "bench_id": "noise-csv-cleaner-2", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_0f5afda14c70", "verdict_id": "verdict_0f5afda14c70", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: csv-cleaner still handles all 13 past tasks.", "narrative": "Passed: csv-cleaner still handles all 13 past tasks. The replay confirmed zero regressions, drift, or uncertain outcomes across every historical pair. Cost was negligible at under one cent for standard models. Nothing to do.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_e8ae735a382c", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1225, "completion_tokens": 142, "usd": 0.00010758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_33ee4ca0f8fe", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 132, "usd": 0.00010602 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_228836f3be49", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 150, "usd": 0.00010998 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_ea4892c6024b", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 105, "usd": 0.00010152 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2511, "completion_tokens": 279, "usd": 0.0010044 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2371, "completion_tokens": 4161, "usd": 0.014854 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_950186207723", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1248, "completion_tokens": 208, "usd": 0.0001248 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2487, "completion_tokens": 296, "usd": 0.0010125 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.85, "prompt_tokens": 2490, "completion_tokens": 2329, "usd": 0.009477 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_b08adb6cff26", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 204, "usd": 0.0001263 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_73bc47c7827d", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 182, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_47c6b01f5ac0", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 230, "usd": 0.00012942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2398, "completion_tokens": 524, "usd": 0.001191 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2615, "completion_tokens": 4445, "usd": 0.01595 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_a32e47b95ba0", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 108, "usd": 0.00010218 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_36ecdca1d7a3", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 122, "usd": 0.00010326 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1b3a21d49d38", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1114, "completion_tokens": 100, "usd": 0.00009084 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_030bed51e051", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 221, "usd": 0.0006105 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1220, "completion_tokens": 1619, "usd": 0.006077 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_0ffe6c11a514", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1187, "completion_tokens": 82, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0013995, "super": 0.0038184, "ultra": 0.046358 }, "counterfactual_ultra_usd": 0.080574, "spent_usd_by_tier": { "nano": 0.00179688, "super": 0.0038184, "ultra": 0.046751 }, "prompt_tokens_by_tier": { "nano": 20892, "super": 249409, "ultra": 8942 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 103.793, "simulated_models": false, "repeat": 2, "error": null }, { "bench_id": "noise-expense-report-2", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_e86bb953c782", "verdict_id": "verdict_e86bb953c782", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The replay confirmed zero regressions, zero drifts, and zero uncertain outcomes across the full evaluation set. Costs remained minimal at fractions of a cent per tier. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_244f58bcaa6b", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1165, "completion_tokens": 131, "usd": 0.00010134 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1408, "completion_tokens": 156, "usd": 0.0005628 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_5ce95cd6c61a", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1417, "completion_tokens": 156, "usd": 0.00012246 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3071, "completion_tokens": 265, "usd": 0.0011598 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_e3e749337852", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1638, "completion_tokens": 100, "usd": 0.00012228 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3645, "completion_tokens": 415, "usd": 0.001467 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3559, "completion_tokens": 1615, "usd": 0.008404 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_a51d59ee3f12", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1289, "completion_tokens": 143, "usd": 0.00011166 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_585ce2c06dfb", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1453, "completion_tokens": 140, "usd": 0.00012078 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_14c73df6e208", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1466, "completion_tokens": 236, "usd": 0.0001446 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_b2aff3e26cce", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1291, "completion_tokens": 148, "usd": 0.00011298 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a4c9f0c53dab", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1320, "completion_tokens": 142, "usd": 0.00011328 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_95365456b991", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1497, "completion_tokens": 172, "usd": 0.0001311 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_149c0bed5fe5", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1447, "completion_tokens": 137, "usd": 0.0001197 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_8c4226da617a", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 155, "usd": 0.00011118 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_d0092945b70f", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 193, "usd": 0.00012258 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_1dcbc7dcf0e4", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 128, "usd": 0.0001047 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_9a7481b6f24a", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 130, "usd": 0.00010554 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_c32583a023eb", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 179, "usd": 0.00011928 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_e4234a3b2dfe", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 89, "usd": 0.00009552 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2489, "completion_tokens": 308, "usd": 0.0010239 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2424, "completion_tokens": 1362, "usd": 0.00651 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_7e0c331fb544", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1277, "completion_tokens": 252, "usd": 0.0001371 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2684, "completion_tokens": 342, "usd": 0.001113 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2653, "completion_tokens": 2910, "usd": 0.011383 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_77a5673ce9cd", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 162, "usd": 0.00011622 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_7e5167168017", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 144, "usd": 0.00010878 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2437, "completion_tokens": 304, "usd": 0.0010047 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2364, "completion_tokens": 3040, "usd": 0.011484 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f8f38923f478", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 99, "usd": 0.00009204 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1344, "completion_tokens": 224, "usd": 0.0006048 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_12077a158c07", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 68, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 220, "usd": 0.0006096 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1193, "completion_tokens": 2387, "usd": 0.008354 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_0f5596654861", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 121, "usd": 0.00010032 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00249924, "super": 0.0075456, "ultra": 0.046135 }, "counterfactual_ultra_usd": 0.109716, "spent_usd_by_tier": { "nano": 0.0034122, "super": 0.0075456, "ultra": 0.046531 }, "prompt_tokens_by_tier": { "nano": 38650, "super": 443154, "ultra": 12433 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 113.572, "simulated_models": false, "repeat": 2, "error": null }, { "bench_id": "noise-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_f39b85e5cd09", "verdict_id": "verdict_f39b85e5cd09", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The replay confirmed zero regressions, zero drift, and zero uncertain pairs across the full evaluation set. Costs remain negligible at fractions of a cent per tier. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_ed2c34df5e23", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 129, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_d36ce896f88b", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1340, "completion_tokens": 160, "usd": 0.0001188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_eda810cc9ac4", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1328, "completion_tokens": 149, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_5cec5575a497", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 92, "usd": 0.000105 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_5b5e17b8a799", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 215, "usd": 0.00013374 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_0504698df97e", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1359, "completion_tokens": 132, "usd": 0.00011322 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_c0cbc69b0314", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1416, "completion_tokens": 127, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_c6ae2464acfb", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1343, "completion_tokens": 178, "usd": 0.0001233 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_e5a7bdca6d52", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 96, "usd": 0.00010872 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_e775ea7cd779", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 133, "usd": 0.00011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_940b70f9ffa9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 91, "usd": 0.00009012 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1344, "completion_tokens": 201, "usd": 0.0005841 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_29ecb82fbfbd", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 68, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 220, "usd": 0.0006096 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1247, "completion_tokens": 867, "usd": 0.003848 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f51b1d84950c", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 107, "usd": 0.00009696 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00143358, "super": 0.0011937, "ultra": 0.003848 }, "counterfactual_ultra_usd": 0.030043, "spent_usd_by_tier": { "nano": 0.00152034, "super": 0.0011937, "ultra": 0.004261 }, "prompt_tokens_by_tier": { "nano": 18343, "super": 151374, "ultra": 1498 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 63.882, "simulated_models": false, "repeat": 2, "error": null }, { "bench_id": "noise-github-triage-2", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_b9f1424237bc", "verdict_id": "verdict_b9f1424237bc", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: github-triage still handles all 13 past tasks.", "narrative": "Passed: github-triage still handles all 13 past tasks. The replay compared current behavior against 13 established cases covering issue classification, routing, and labeling logic. Every case produced the expected output with zero regressions, drift, or uncertain results. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_01eb45e7f72d", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 152, "usd": 0.00010626 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_430ff366a63c", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 147, "usd": 0.00010758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_6b5530ae52c4", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 138, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_60e391f609d3", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 101, "usd": 0.00010662 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_f2bfc073d095", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 167, "usd": 0.00011922 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_aaa2cdace933", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 131, "usd": 0.0001158 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2952, "completion_tokens": 298, "usd": 0.0011538 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2864, "completion_tokens": 1473, "usd": 0.007283 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_2ef72a6e5275", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1404, "completion_tokens": 138, "usd": 0.00011736 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2813, "completion_tokens": 356, "usd": 0.0011643 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_aee53755701c", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1334, "completion_tokens": 116, "usd": 0.00010788 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_988415973823", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1402, "completion_tokens": 136, "usd": 0.00011676 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_85c8a41e8c22", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1364, "completion_tokens": 123, "usd": 0.00011136 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_9eb3a63de58a", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 123, "usd": 0.0000978 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1344, "completion_tokens": 201, "usd": 0.0005841 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e5f6af5ae207", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 218, "usd": 0.0006078 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 1124, "usd": 0.004607 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c5ef3b5e01c2", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 82, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00139974, "super": 0.00351, "ultra": 0.01189 }, "counterfactual_ultra_usd": 0.045293, "spent_usd_by_tier": { "nano": 0.00164388, "super": 0.00351, "ultra": 0.012331 }, "prompt_tokens_by_tier": { "nano": 19354, "super": 214159, "ultra": 4342 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 77.245, "simulated_models": false, "repeat": 2, "error": null } ] }, "cascade": { "config": { "name": "cascade", "description": "Full cascade — Level 0, Nano screen, Super check, Ultra adjudication, drift check.", "experiments": [ "E1", "E2", "E3", "E4", "E5" ], "mode": "edits", "cascade": { "mode": "cascade", "level0": true, "drift": true, "controls": true }, "single_judge_blocks": null, "types": [ "R1", "R2", "R3", "R4", "R5", "R6", "N0", "D1" ], "noise_repeats": 3, "runner": "local", "concurrency": 2, "reuse_replays": null, "max_traces": 40 }, "config_path": "bench/configs/cascade.yaml", "command": "ratchet bench run --config bench/configs/cascade.yaml --out bench/results/2026-10-16 --types D1 --limit 2 --yes", "started_at": "2026-10-16T12:14:41.898273Z", "updated_at": "2026-10-16T13:56:45.647250Z", "edits": [ { "bench_id": "n0-github-triage-2", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_632e6d7d6113", "verdict_id": "verdict_632e6d7d6113", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The change only rewords text in the skill file without altering any step, tool, argument, or rule. Twelve replays passed and one was marked uncertain because the judge could not determine if the cosmetic edits affected behavior. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_811b35141f2c", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 99, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_b3d50ce1b799", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1217, "completion_tokens": 135, "usd": 0.00010542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_7741c40ef280", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 157, "usd": 0.00011898 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_abe3d0283a76", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 148, "usd": 0.0001179 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_a0217d5141b7", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 105, "usd": 0.00010506 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_f5d663dbc019", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only rewords text (e.g., 'when' to 'whenever', 'Steps' to 'Procedure') without changing any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 4829, "completion_tokens": 900, "usd": 0.00050574 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3314, "completion_tokens": 301, "usd": 0.0012651 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 3212, "completion_tokens": 3008, "usd": 0.012236 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7e7f0890e15f", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1370, "completion_tokens": 179, "usd": 0.00012516 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2975, "completion_tokens": 321, "usd": 0.0011814 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a1def1b764dc", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 110, "usd": 0.00010716 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_0490537a6338", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 174, "usd": 0.00012516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_8b534e552950", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 195, "usd": 0.00012966 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_7e792837459f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 96, "usd": 0.00009132 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_95212662e369", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 68, "usd": 0.00008508 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1573, "completion_tokens": 217, "usd": 0.0006672 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1464, "completion_tokens": 1556, "usd": 0.006132 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a24da7604ca9", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 135, "usd": 0.00010296 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00181242, "super": 0.0031137, "ultra": 0.018368 }, "counterfactual_ultra_usd": 0.056453, "spent_usd_by_tier": { "nano": 0.00201714, "super": 0.0031137, "ultra": 0.018903 }, "prompt_tokens_by_tier": { "nano": 22719, "super": 224487, "ultra": 4989 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 94.649, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-2", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_8b1408376d8d", "verdict_id": "verdict_8b1408376d8d", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The edit changed one line in SKILL.md and the regression suite ran 22 pairs with zero regressions, zero drifts, and zero uncertain results. Every previously verified behavior remains intact. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_db2fa52f33ec", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1106, "completion_tokens": 112, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_663c04fc7679", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 196, "usd": 0.00013278 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_b8eacf7ca9eb", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1637, "completion_tokens": 132, "usd": 0.0001299 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3975, "completion_tokens": 142, "usd": 0.0013203 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3968, "completion_tokens": 2635, "usd": 0.011873 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_df4a9a6d2d5e", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 193, "usd": 0.00012294 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_2aba1c6b9a48", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 114, "usd": 0.0001131 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_4d8a32096183", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 179, "usd": 0.00012948 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_5d79f77bf29a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1315, "completion_tokens": 192, "usd": 0.00012498 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_fa5cfd551a14", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 154, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_bcd2f6c4de1b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 188, "usd": 0.00013638 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6c9bfaed6179", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 133, "usd": 0.00011946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_979bc36816b3", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 168, "usd": 0.00011286 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_0444b760bf07", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 194, "usd": 0.00012282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_997e188d6e47", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 164, "usd": 0.00011406 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_d40c1503ab73", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 170, "usd": 0.0001164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_8111d388d469", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 213, "usd": 0.00012816 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0727abbc4d2a", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 220, "usd": 0.00012696 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2772, "completion_tokens": 294, "usd": 0.0010962 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_cfff2f4aa409", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 146, "usd": 0.00010998 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2328, "completion_tokens": 305, "usd": 0.0009729 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_52606712bd67", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 218, "usd": 0.00012822 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_542487f91fa5", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 178, "usd": 0.00011766 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2698, "completion_tokens": 334, "usd": 0.00111 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 5654, "completion_tokens": 5491, "usd": 0.022127 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_599436fdbbc6", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 141, "usd": 0.00010284 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_de6195626ea9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1122, "completion_tokens": 73, "usd": 0.00008484 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1722, "completion_tokens": 221, "usd": 0.0007155 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1601, "completion_tokens": 1000, "usd": 0.004601 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e212cacce84", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 82, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0025734, "super": 0.0052149, "ultra": 0.038601 }, "counterfactual_ultra_usd": 0.095314, "spent_usd_by_tier": { "nano": 0.00289104, "super": 0.0052149, "ultra": 0.03904 }, "prompt_tokens_by_tier": { "nano": 32728, "super": 377841, "ultra": 11464 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 108.145, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-1", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_e5d12dd4c52e", "verdict_id": "verdict_e5d12dd4c52e", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now fails \"asks for details for #91\" on 5 of 13 past tasks.", "narrative": "Blocked: github-triage now fails \"asks for details for #91\" on 5 of 13 past tasks. The edit changed SKILL.md and triggered a replay of 13 triage tasks. Six passed, but five regressed because the agent stopped asking for required details on issues #91, #36, #15, #64, and the docs-site task. Two additional tasks were held for human review due to judge disagreements and a labeling mismatch on issue #502. The regressions are consistent and specific to the detail-requesting behavior. The change was automatically reverted. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_29198b3e615a", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 126, "usd": 0.00009858 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_ae5049772f22", "task_id": "tri-status-page", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Judges disagreed: Super found a regression and Ultra did not. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1175, "completion_tokens": 127, "usd": 0.00010098 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2395, "completion_tokens": 220, "usd": 0.0009165 }, { "tier": "ultra", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2301, "completion_tokens": 2337, "usd": 0.009312 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_6a4a7b2c3a20", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay violates the success criterion 'labels issue #502 as question' (it labels #502 as 'enhancement' instead). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1365, "completion_tokens": 146, "usd": 0.00011694 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3185, "completion_tokens": 290, "usd": 0.0012165 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3159, "completion_tokens": 1751, "usd": 0.008412 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ff7547adf9e1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1364, "completion_tokens": 134, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_95dc2c9a2266", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "asks for details for #91 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1286, "completion_tokens": 139, "usd": 0.00011052 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2979, "completion_tokens": 249, "usd": 0.0011178 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2864, "completion_tokens": 1651, "usd": 0.007817 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_1ac70e25ffb3", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1353, "completion_tokens": 129, "usd": 0.00011214 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3138, "completion_tokens": 301, "usd": 0.0012123 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3014, "completion_tokens": 3850, "usd": 0.014564 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_3be8109cfb2f", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states that no details were asked for is no longer met.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 176, "usd": 0.0001254 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3134, "completion_tokens": 263, "usd": 0.0011769 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3089, "completion_tokens": 1153, "usd": 0.006548 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_650e3f7e3ede", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "asks for details on #64 and cites github.comment as artifact changed is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1308, "completion_tokens": 163, "usd": 0.0001176 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3030, "completion_tokens": 262, "usd": 0.0011448 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2958, "completion_tokens": 1401, "usd": 0.007161 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_e5bee22bc04e", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "mentions that issue #36 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1400, "completion_tokens": 96, "usd": 0.00010704 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3411, "completion_tokens": 284, "usd": 0.0012789 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3286, "completion_tokens": 1389, "usd": 0.007453 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_0c922bfa825f", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "mentions #15 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 93, "usd": 0.00010536 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3130, "completion_tokens": 302, "usd": 0.0012108 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2997, "completion_tokens": 1151, "usd": 0.00645 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_00c8cee98bc3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 91, "usd": 0.00009012 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_05517158aa56", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1557, "completion_tokens": 219, "usd": 0.0006642 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 1445, "completion_tokens": 852, "usd": 0.004001 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_bdbe9648c85c", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 99, "usd": 0.0000957 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00138114, "super": 0.0099387, "ultra": 0.071718 }, "counterfactual_ultra_usd": 0.126275, "spent_usd_by_tier": { "nano": 0.00242244, "super": 0.0099387, "ultra": 0.072631 }, "prompt_tokens_by_tier": { "nano": 28694, "super": 366697, "ultra": 25630 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 139.91, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_d15aa27b3ad3", "verdict_id": "verdict_d15aa27b3ad3", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system replayed 13 test pairs against the updated fx-expense-converter skill; 12 passed and 1 was uncertain. The uncertain case, \"Convert at a weekend date,\" failed the criterion requiring the correct date 2026-10-11 because the replay showed 2026-10-09 in the rate line, and Tavily detected the exchange rate value shifting from 1.0768 to 1.0800. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_885b30526626", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1347, "completion_tokens": 160, "usd": 0.00011922 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2512, "completion_tokens": 323, "usd": 0.0010443 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_fcf301073639", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails the 'uses the correct date 2026-10-11' criterion because it shows 2026-10-09 in the rate line (m003). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1378, "completion_tokens": 172, "usd": 0.00012396 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2758, "completion_tokens": 289, "usd": 0.0010875 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2748, "completion_tokens": 2094, "usd": 0.00903 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_82a70ccb2af8", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1351, "completion_tokens": 187, "usd": 0.00012594 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2612, "completion_tokens": 303, "usd": 0.0010563 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_973ca2670ff2", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1405, "completion_tokens": 100, "usd": 0.0001083 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2734, "completion_tokens": 354, "usd": 0.0011388 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3afca948a379", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 257, "usd": 0.00014664 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2573, "completion_tokens": 347, "usd": 0.0010842 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_f5be008e5f9d", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 170, "usd": 0.00012522 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2751, "completion_tokens": 332, "usd": 0.0011241 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_b55b161ede07", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1474, "completion_tokens": 138, "usd": 0.00012156 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2758, "completion_tokens": 373, "usd": 0.0011631 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_e70094e8923d", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1354, "completion_tokens": 244, "usd": 0.0001398 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2561, "completion_tokens": 309, "usd": 0.0010464 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5bde00727c59", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1475, "completion_tokens": 118, "usd": 0.00011682 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2993, "completion_tokens": 307, "usd": 0.0011742 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_96be56214418", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1435, "completion_tokens": 219, "usd": 0.00013866 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2746, "completion_tokens": 309, "usd": 0.0011019 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d1a68d58b4ae", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 104, "usd": 0.00009396 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1671, "completion_tokens": 222, "usd": 0.0007011 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_96c18d79778a", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 69, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1723, "completion_tokens": 221, "usd": 0.0007158 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1591, "completion_tokens": 1236, "usd": 0.005299 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9cb3c3858cd1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00153846, "super": 0.0124377, "ultra": 0.014329 }, "counterfactual_ultra_usd": 0.079409, "spent_usd_by_tier": { "nano": 0.00179406, "super": 0.0126285, "ultra": 0.015075 }, "prompt_tokens_by_tier": { "nano": 20761, "super": 196865, "ultra": 4683 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 70.99, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-1", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_fece588f0817", "verdict_id": "verdict_fece588f0817", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would close issues in 5 of 13 replays.", "narrative": "Blocked: the new github-triage would close issues in 5 of 13 replays. The edit changed SKILL.md and Ratchet replayed 13 triage tasks against the original runs. Eight passed, but five regressed because the updated skill attempted new calls to github.close_issue and github.comment that the sandbox rejected. The affected replays are acme/search-indexer, acme/infra-terraform, acme/auth-service, acme/docs-site, and acme/mobile-app. The changes introduce unauthorized side effects that would mutate GitHub state. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_f8c354ae33e9", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 150, "usd": 0.0001029 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_478d9ee2ce19", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 135, "usd": 0.0001047 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_755deee2dac3", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ce4b9acb3157", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 3 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 3, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_11ee7c7f4f30", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_9432a20fd19d", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1419, "completion_tokens": 161, "usd": 0.00012378 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3381, "completion_tokens": 279, "usd": 0.0012654 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3286, "completion_tokens": 2440, "usd": 0.010606 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_715dfb3b7eec", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_5ba0f0c4e92d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_39066d273f98", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1415, "completion_tokens": 182, "usd": 0.00012858 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3566, "completion_tokens": 402, "usd": 0.0014316 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_c944d9078bcc", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1396, "completion_tokens": 127, "usd": 0.00011424 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3320, "completion_tokens": 344, "usd": 0.0013056 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_56c2b63686fb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 95, "usd": 0.00009324 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1600, "completion_tokens": 218, "usd": 0.0006762 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a8e17ca9255f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1616, "completion_tokens": 219, "usd": 0.0006819 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 1474, "completion_tokens": 2279, "usd": 0.008311 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_dd5897c02420", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 122, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0008547, "super": 0.0053607, "ultra": 0.018917 }, "counterfactual_ultra_usd": 0.049987, "spent_usd_by_tier": { "nano": 0.001638, "super": 0.0053607, "ultra": 0.019792 }, "prompt_tokens_by_tier": { "nano": 19372, "super": 367264, "ultra": 5266 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 115.657, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-1", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_3d5beec38990", "verdict_id": "verdict_3d5beec38990", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail", "agreed_regression" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system ran 13 replay pairs against the proposed change to SKILL.md. Ten passed, two regressed, and one was uncertain. The clearest failure was on acme/docs-site, where the agent made a new github.comment call that the sandbox rejected. A second regression on acme/status-page dropped a required output constraint. The uncertain case on acme/data-pipeline matched the golden output exactly but was held for human review. The edit changed only one line in the skill file. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_165175858316", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 124, "usd": 0.00009954 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_187d5cdcf6dc", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "contains no additional text that is not part of the required output format is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1228, "completion_tokens": 129, "usd": 0.00010464 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2602, "completion_tokens": 221, "usd": 0.0009795 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2525, "completion_tokens": 1100, "usd": 0.005825 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3f0a0fcb0075", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 182, "usd": 0.00012528 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_9972d203f927", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1361, "completion_tokens": 137, "usd": 0.00011454 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c71e5c2370b8", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 128, "usd": 0.00010824 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3017, "completion_tokens": 287, "usd": 0.0011634 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_eb9d9e6851a5", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay final output matches the golden final output exactly and meets all textual success criteria (issue count, issue list with types, #406 noted as labeled, no details requested). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 152, "usd": 0.00011952 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3101, "completion_tokens": 299, "usd": 0.0011994 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3037, "completion_tokens": 3304, "usd": 0.012949 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_de7759ea9e17", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_092b5ebe7778", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1350, "completion_tokens": 142, "usd": 0.00011508 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3084, "completion_tokens": 289, "usd": 0.0011853 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_968755dee57c", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1413, "completion_tokens": 174, "usd": 0.00012654 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_10dca42d0cf4", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1392, "completion_tokens": 127, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_990c76131fc0", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_47c34378e66c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1182, "completion_tokens": 141, "usd": 0.00010476 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_602f5ee8bbac", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 84, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0013182, "super": 0.0045276, "ultra": 0.018774 }, "counterfactual_ultra_usd": 0.054218, "spent_usd_by_tier": { "nano": 0.001677, "super": 0.0045276, "ultra": 0.019554 }, "prompt_tokens_by_tier": { "nano": 19466, "super": 278830, "ultra": 5961 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 98.884, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_c5b788d41600", "verdict_id": "verdict_c5b788d41600", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 14 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 14 replays. Review before merging. The skill diff only rewords the description without changing tool calls or logic. Twelve replays passed. Two were held for human review: a weekend conversion where Tavily detected a rate shift from 1.0768 to 1.0800, and a formal rewrite where the replay added a greeting and extra wording absent from the golden output. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_9b9fe0986841", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1324, "completion_tokens": 110, "usd": 0.00010584 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_f97cae02fbc4", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only rewords the description from 'foreign-currency amounts' to 'foreign-currency expense amounts' without changing any tool calls, logic, or output formatting. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1352, "completion_tokens": 120, "usd": 0.00010992 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2555, "completion_tokens": 285, "usd": 0.001023 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2534, "completion_tokens": 2236, "usd": 0.009242 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6eb01a2d41d3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1342, "completion_tokens": 156, "usd": 0.00011796 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_315104ec0683", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 100, "usd": 0.00010692 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_762ca612dbe8", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 228, "usd": 0.00013614 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_4ef2ca8f8b72", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1396, "completion_tokens": 162, "usd": 0.00012264 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_53e1872e54f8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 147, "usd": 0.00012168 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_fddd61f372fe", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 155, "usd": 0.00011634 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_bda2112c7543", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 121, "usd": 0.00011616 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c3051db65b47", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 219, "usd": 0.00013728 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_8981ccff882f", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1650, "completion_tokens": 137, "usd": 0.00013188 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3818, "completion_tokens": 168, "usd": 0.0012966 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3809, "completion_tokens": 2050, "usd": 0.009959 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_ec291cce2b97", "task_id": "canary-formal-rewrite", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output includes a greeting and extra wording not present in the golden output, violating the criterion against additional content (replay step m001 vs golden step m001). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 104, "usd": 0.0000942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1514, "completion_tokens": 237, "usd": 0.0006675 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1419, "completion_tokens": 2850, "usd": 0.009969 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_cbb67ab2491c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 152, "usd": 0.00010668 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1533, "completion_tokens": 214, "usd": 0.0006525 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1466, "completion_tokens": 892, "usd": 0.004142 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_91fcb24262ed", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 118, "usd": 0.00010026 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0016239, "super": 0.0036396, "ultra": 0.033312 }, "counterfactual_ultra_usd": 0.07048, "spent_usd_by_tier": { "nano": 0.00209352, "super": 0.0038295, "ultra": 0.03404 }, "prompt_tokens_by_tier": { "nano": 24912, "super": 223058, "ultra": 9623 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 80.594, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_9886f90248a6", "verdict_id": "verdict_9886f90248a6", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs for the fx-expense-converter skill after a documentation-only change to SKILL.md. Twelve pairs passed and one was marked uncertain. The uncertain case, \"Compute 17% of 240,\" produced the correct numeric result and required formula but was held because the output formatting differed slightly from the expected bold styling. The change did not introduce any regressions or drift. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_382f2e4d44ae", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1300, "completion_tokens": 110, "usd": 0.0001044 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_75e2cc6a76a8", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 125, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_c536f0cd0389", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 171, "usd": 0.00012 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_a8c493d9890b", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 114, "usd": 0.00011028 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_b7a6a78b225c", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1345, "completion_tokens": 191, "usd": 0.00012654 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a00a986b53f7", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1383, "completion_tokens": 184, "usd": 0.00012714 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_c7bcfb7bf68e", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 137, "usd": 0.00011856 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_3b832d8b5c25", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1367, "completion_tokens": 158, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_f76c8a253ef0", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 118, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c432f688c91f", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 257, "usd": 0.00014568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1f1fa89dddfb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_9470f1a93d4f", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output '17% of 240 = 0.17 × 240 = 40.8.' meets all success criteria: it contains the numeric result 40.8, includes '17% of 240', includes '= 0.17 × 240', and includes '= 40.8' (the bold formatting difference is explicitly not a r…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 72, "usd": 0.00008532 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1592, "completion_tokens": 245, "usd": 0.0006981 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 1501, "completion_tokens": 1275, "usd": 0.005326 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f86b4f70f4f6", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 106, "usd": 0.00009522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00147216, "super": 0.0006981, "ultra": 0.005326 }, "counterfactual_ultra_usd": 0.030348, "spent_usd_by_tier": { "nano": 0.00157428, "super": 0.0006981, "ultra": 0.006053 }, "prompt_tokens_by_tier": { "nano": 18330, "super": 150230, "ultra": 1877 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 62.194, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-1", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_a9f54a4073f9", "verdict_id": "verdict_a9f54a4073f9", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 22 replays. Review before merging. The system replayed 22 test pairs for the expense-report skill change; 21 passed and 1 was marked uncertain. The uncertain case, \"Clean support/tickets.csv,\" produced a file output identical to the golden run with correct row counts, headers, and no duplicate or empty IDs, yet the judge withheld automatic approval. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_ddb84b0fe47b", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1178, "completion_tokens": 127, "usd": 0.00010116 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1647, "completion_tokens": 175, "usd": 0.0006516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_7d67f2353f20", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 115, "usd": 0.00011202 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3166, "completion_tokens": 286, "usd": 0.0012072 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_acc26084d722", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1658, "completion_tokens": 129, "usd": 0.00013044 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3971, "completion_tokens": 134, "usd": 0.0013119 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3953, "completion_tokens": 2766, "usd": 0.012251 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_af5ade88f377", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 181, "usd": 0.0001215 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2541, "completion_tokens": 271, "usd": 0.0010062 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_a69ed051d7f4", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1441, "completion_tokens": 130, "usd": 0.00011766 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3242, "completion_tokens": 300, "usd": 0.0012426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_3e7d7797755d", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1430, "completion_tokens": 211, "usd": 0.00013644 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3239, "completion_tokens": 301, "usd": 0.0012426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_079c36312676", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1303, "completion_tokens": 201, "usd": 0.00012642 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2566, "completion_tokens": 273, "usd": 0.0010155 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_f07720304e5f", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1296, "completion_tokens": 168, "usd": 0.00011808 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2555, "completion_tokens": 273, "usd": 0.0010122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_8626f6fe5abb", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1521, "completion_tokens": 155, "usd": 0.00012846 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3365, "completion_tokens": 410, "usd": 0.0013785 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_521e59e6091f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1435, "completion_tokens": 132, "usd": 0.00011778 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3257, "completion_tokens": 310, "usd": 0.0012561 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_e31167be7710", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 184, "usd": 0.0001167 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_229e9b0e57cc", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 190, "usd": 0.00012186 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_1aec4a330ef6", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 158, "usd": 0.00011262 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_5b8721009163", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 155, "usd": 0.00011154 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3aee4f6f9c1d", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay's actual file output (t003) is identical to the golden run's file output and meets all success criteria: 3 data rows, 1 header, no duplicate IDs, no empty IDs, all required rows present. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 101, "usd": 0.00010128 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2742, "completion_tokens": 280, "usd": 0.0010746 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.6, "prompt_tokens": 2659, "completion_tokens": 2761, "usd": 0.010942 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_838608cb3f0e", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 225, "usd": 0.00013104 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2693, "completion_tokens": 319, "usd": 0.001095 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_010f6ae26a1d", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 235, "usd": 0.00013206 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2252, "completion_tokens": 265, "usd": 0.0009141 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_a8ba615a5d23", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1313, "completion_tokens": 125, "usd": 0.00010878 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_22e0e0ff85d9", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 181, "usd": 0.00011838 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2774, "completion_tokens": 339, "usd": 0.0011373 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2748, "completion_tokens": 4009, "usd": 0.014775 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_15046a030360", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 97, "usd": 0.00009228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3ac334e2f481", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1621, "completion_tokens": 233, "usd": 0.000696 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1514, "completion_tokens": 1139, "usd": 0.004931 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb845c98bd90", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1175, "completion_tokens": 121, "usd": 0.00009954 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0025428, "super": 0.0162414, "ultra": 0.042899 }, "counterfactual_ultra_usd": 0.136024, "spent_usd_by_tier": { "nano": 0.00300072, "super": 0.0162414, "ultra": 0.043538 }, "prompt_tokens_by_tier": { "nano": 34124, "super": 428558, "ultra": 11192 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 95.272, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-expense-report-1", "skill": "expense-report", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_760317e77b11", "verdict_id": "verdict_760317e77b11", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 5 of 22 past tasks.", "narrative": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 5 of 22 past tasks. The edit changed SKILL.md and triggered regressions on five test cases covering bakery, software, office supplies, conference, and lunch plus train receipts where required fields disappeared. One coworking and courier case was uncertain with judges split. Sixteen cases still pass. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_73fd927ba82e", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 139, "usd": 0.0001026 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1665, "completion_tokens": 155, "usd": 0.000639 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_71599b6cf2f5", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 221, "usd": 0.00013698 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3359, "completion_tokens": 280, "usd": 0.0012597 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3298, "completion_tokens": 1874, "usd": 0.00892 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_68029dbf8d42", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1621, "completion_tokens": 139, "usd": 0.00013062 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3802, "completion_tokens": 435, "usd": 0.0015321 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3808, "completion_tokens": 2788, "usd": 0.012172 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_0061479721a1", "task_id": "exp-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the VAT 18.81 EUR is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1293, "completion_tokens": 128, "usd": 0.0001083 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2488, "completion_tokens": 274, "usd": 0.000993 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2392, "completion_tokens": 802, "usd": 0.004798 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_cde84ba8b5d5", "task_id": "exp-coworking-courier", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Judges disagreed: Super found a regression and Ultra did not. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1427, "completion_tokens": 122, "usd": 0.0001149 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3176, "completion_tokens": 322, "usd": 0.0012426 }, { "tier": "ultra", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3147, "completion_tokens": 1805, "usd": 0.008562 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_d2bfe3e90095", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 136, "usd": 0.00011754 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3203, "completion_tokens": 313, "usd": 0.0012426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_ea8d1c2b4f4b", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the VAT amount '7.58 EUR' is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 121, "usd": 0.00010608 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2533, "completion_tokens": 268, "usd": 0.0010011 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.98, "prompt_tokens": 2400, "completion_tokens": 1049, "usd": 0.005547 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_bcff8a71d40d", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the VAT as 79.83 EUR is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1312, "completion_tokens": 144, "usd": 0.00011328 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2525, "completion_tokens": 286, "usd": 0.0010149 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2497, "completion_tokens": 1078, "usd": 0.005731 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_49a1a41ef5ee", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1507, "completion_tokens": 140, "usd": 0.00012402 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3322, "completion_tokens": 384, "usd": 0.0013422 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3304, "completion_tokens": 1409, "usd": 0.007531 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_87ffd7a9b61f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1444, "completion_tokens": 114, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_bfab509e3676", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1257, "completion_tokens": 153, "usd": 0.00011214 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_ae36c7d648f7", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 177, "usd": 0.00011658 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_c94efe16dd1d", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 140, "usd": 0.0001083 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ffd4fd935f19", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1263, "completion_tokens": 187, "usd": 0.00012066 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_699475671d46", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 133, "usd": 0.00010824 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_bbc5ac91d665", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 169, "usd": 0.00011688 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2718, "completion_tokens": 306, "usd": 0.0010908 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_657e87d6ef5e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 193, "usd": 0.00012198 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2258, "completion_tokens": 265, "usd": 0.0009159 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_8d54d0935d9d", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 199, "usd": 0.0001251 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_cbe81857d988", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 122, "usd": 0.0001035 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2578, "completion_tokens": 327, "usd": 0.0010677 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2557, "completion_tokens": 4665, "usd": 0.016552 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_db46f3b0746f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 147, "usd": 0.00010356 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1587, "completion_tokens": 221, "usd": 0.000675 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_10a2feb40124", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1627, "completion_tokens": 214, "usd": 0.0006807 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1489, "completion_tokens": 1608, "usd": 0.006313 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_da322788c4ba", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 104, "usd": 0.0000969 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00248892, "super": 0.0146973, "ultra": 0.076126 }, "counterfactual_ultra_usd": 0.163399, "spent_usd_by_tier": { "nano": 0.00353304, "super": 0.0146973, "ultra": 0.076887 }, "prompt_tokens_by_tier": { "nano": 40972, "super": 489002, "ultra": 25371 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 122.405, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-2", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_a65572ab314a", "verdict_id": "verdict_a65572ab314a", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail", "agreed_regression" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The edit changed one line in SKILL.md. Thirteen replays ran: ten passed, two regressed, one was uncertain. The docs-site replay added a github.comment call the original run never made, which the sandbox rejected. The billing-api replay dropped a required request for details on issue #36. The data-pipeline replay met all success criteria but was held for human review. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5bbf428edf52", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 126, "usd": 0.00009828 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1606, "completion_tokens": 177, "usd": 0.0006411 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f2d56880f7aa", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 145, "usd": 0.0001071 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_c651c351049c", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 147, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_b6ff0cb2781b", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac63d98c8094", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1314, "completion_tokens": 154, "usd": 0.0001158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_537a1694bbb0", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets success criteria 1-5: it states 4 open issues (m004), lists each with type (m004), mentions #406 already labeled (m004), states no details requested (m004), and shows github.label artifacts changed (t002-t004). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1395, "completion_tokens": 202, "usd": 0.00013218 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3371, "completion_tokens": 311, "usd": 0.0012912 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3341, "completion_tokens": 4190, "usd": 0.015911 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_9e2fa3314653", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d200a62ffe51", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 100, "usd": 0.00010476 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_465ab5c43f9c", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "mentions that issue #36 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1401, "completion_tokens": 102, "usd": 0.00010854 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3357, "completion_tokens": 333, "usd": 0.0013068 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3284, "completion_tokens": 1680, "usd": 0.008324 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_19dcfee12b9c", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 117, "usd": 0.00011208 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f59a42a6f25f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f5acfa748023", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1626, "completion_tokens": 223, "usd": 0.0006885 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1502, "completion_tokens": 1806, "usd": 0.00692 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_de28baa02c6f", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 82, "usd": 0.00009234 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00116916, "super": 0.0039276, "ultra": 0.031155 }, "counterfactual_ultra_usd": 0.062386, "spent_usd_by_tier": { "nano": 0.00164982, "super": 0.0039276, "ultra": 0.031924 }, "prompt_tokens_by_tier": { "nano": 19421, "super": 296016, "ultra": 8554 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 104.862, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-2", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_67bc03fe6084", "verdict_id": "verdict_67bc03fe6084", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The skill change only adds a required \"Closed:\" output format line and does not alter triage labeling rules. All 12 other replays pass. One replay for acme/mobile-app failed to label issue #58 as a bug, but the judge marked this as uncertain rather than a regression. The cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0a31fbd0f6ab", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 150, "usd": 0.00010434 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e3ade3806218", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1185, "completion_tokens": 146, "usd": 0.00010614 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e7ec930d750a", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1359, "completion_tokens": 159, "usd": 0.0001197 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_c0c9ec6a7dc8", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1353, "completion_tokens": 150, "usd": 0.00011718 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3257, "completion_tokens": 267, "usd": 0.0012174 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_9484d74e29ab", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1330, "completion_tokens": 138, "usd": 0.00011292 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3022, "completion_tokens": 252, "usd": 0.0011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_dcfc05cb8a6e", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1403, "completion_tokens": 169, "usd": 0.00012474 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3125, "completion_tokens": 396, "usd": 0.0012939 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_b752b4bba738", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 200, "usd": 0.00013338 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3006, "completion_tokens": 290, "usd": 0.0011628 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_811329a74d4f", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails to label issue #58 as bug (criterion 2), but the skill diff only adds an output format requirement for 'Closed:' and does not change triage labeling rules. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1353, "completion_tokens": 151, "usd": 0.00011742 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 2985, "completion_tokens": 286, "usd": 0.0011529 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2919, "completion_tokens": 2037, "usd": 0.00903 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9dd1c1c9ae58", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 157, "usd": 0.00012054 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3625, "completion_tokens": 295, "usd": 0.001353 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d9834ffa3e3a", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1463, "completion_tokens": 118, "usd": 0.0001161 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3343, "completion_tokens": 293, "usd": 0.0012666 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3f0ff18aec4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 107, "usd": 0.00009468 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1497, "completion_tokens": 218, "usd": 0.0006453 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_5b3dba8d5830", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 64, "usd": 0.00008556 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1537, "completion_tokens": 219, "usd": 0.0006582 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.6, "prompt_tokens": 1437, "completion_tokens": 1438, "usd": 0.005751 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f28450be2e37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 104, "usd": 0.0000969 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014496, "super": 0.0098835, "ultra": 0.014781 }, "counterfactual_ultra_usd": 0.070073, "spent_usd_by_tier": { "nano": 0.0016368, "super": 0.0098835, "ultra": 0.015399 }, "prompt_tokens_by_tier": { "nano": 19400, "super": 250086, "ultra": 4671 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 77.46, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_2a9f31c895fd", "verdict_id": "verdict_2a9f31c895fd", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs against the updated fx-expense-converter skill and found 12 passing with no regressions, drift, or improvements. One case, \"Compute 17% of 240,\" produced an uncertain result because the skill change did not explain the output difference, suggesting possible nondeterminism. That specific replay was held for human review. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_bf3fa0074e07", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 132, "usd": 0.0001104 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_e649669b0a50", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 160, "usd": 0.00011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6b67a590cae3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1340, "completion_tokens": 190, "usd": 0.000126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_1f156a1c8f92", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1370, "completion_tokens": 102, "usd": 0.00010668 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3e98cef87a9d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 223, "usd": 0.00013494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9ffedc5fd087", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 159, "usd": 0.00012048 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_941456f71c6b", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1439, "completion_tokens": 136, "usd": 0.00011898 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_4edc319d8099", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 194, "usd": 0.00012786 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5ece1e8c9118", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 97, "usd": 0.0001104 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_400354316051", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1388, "completion_tokens": 141, "usd": 0.00011712 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_6cfa1d232eb4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_13feb5997c98", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1543, "completion_tokens": 218, "usd": 0.0006591 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 1309, "usd": 0.005334 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e2f6c6c99b4", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1199, "completion_tokens": 106, "usd": 0.00009738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00146616, "super": 0.0006591, "ultra": 0.005334 }, "counterfactual_ultra_usd": 0.030157, "spent_usd_by_tier": { "nano": 0.00156276, "super": 0.0006591, "ultra": 0.005966 }, "prompt_tokens_by_tier": { "nano": 18354, "super": 150002, "ultra": 1706 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 60.772, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-2", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_b63aa8f65e1f", "verdict_id": "verdict_b63aa8f65e1f", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The csv-cleaner skill documentation was edited with two additions and two deletions. Automated judges replayed 13 test pairs: 12 passed, 1 was uncertain, and none regressed, drifted, or improved. The uncertain case, \"Compute 17% of 240,\" showed a difference the skill diff does not explain, likely due to nondeterminism. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_959e7e0816aa", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 185, "usd": 0.00011862 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2239, "completion_tokens": 294, "usd": 0.0009363 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_b81786a373a5", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 135, "usd": 0.00010746 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_1dca9722e993", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 168, "usd": 0.00011286 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_96dcc8778ae9", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 164, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_130fa3316eae", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 140, "usd": 0.00010992 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2722, "completion_tokens": 295, "usd": 0.0010821 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2626, "completion_tokens": 1448, "usd": 0.00697 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_b7dba255cdef", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 178, "usd": 0.00011862 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_fe6d06e8fe47", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 201, "usd": 0.00012234 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_5d23e89670d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 191, "usd": 0.00012006 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2606, "completion_tokens": 309, "usd": 0.0010599 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2572, "completion_tokens": 4175, "usd": 0.015097 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_02fdaa1af68e", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1275, "completion_tokens": 203, "usd": 0.00012522 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3334, "completion_tokens": 270, "usd": 0.0012432 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7c626553532c", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 154, "usd": 0.00011166 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8b65fb6e74ff", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 110, "usd": 0.0000954 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1493, "completion_tokens": 223, "usd": 0.0006486 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f33745e12a8e", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1569, "completion_tokens": 219, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1469, "completion_tokens": 1062, "usd": 0.004655 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_690661728b24", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 82, "usd": 0.00008946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00143406, "super": 0.0056379, "ultra": 0.026722 }, "counterfactual_ultra_usd": 0.067433, "spent_usd_by_tier": { "nano": 0.00173064, "super": 0.0056379, "ultra": 0.027371 }, "prompt_tokens_by_tier": { "nano": 19600, "super": 252952, "ultra": 6962 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 88.731, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-expense-report-1", "skill": "expense-report", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_8f3c11ab9747", "verdict_id": "verdict_8f3c11ab9747", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The edit changed one line in SKILL.md and the full regression suite ran clean with zero regressions, drift, or uncertain results across both nano and super tiers. Ultra tier confirmed the same behavior at higher cost. No tasks improved or degraded. The change is semantically neutral and the skill remains stable. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_60c12e465a52", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1129, "completion_tokens": 140, "usd": 0.00010134 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1588, "completion_tokens": 158, "usd": 0.0006186 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_87b3a1580656", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 189, "usd": 0.00012852 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_ac01bb0f765a", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1602, "completion_tokens": 141, "usd": 0.00012996 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3794, "completion_tokens": 417, "usd": 0.0015135 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3784, "completion_tokens": 1617, "usd": 0.008635 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_ba09cf46ac3d", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 178, "usd": 0.00011934 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_ca6090d7b4f3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1453, "completion_tokens": 174, "usd": 0.00012894 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_f88f40b0fdd4", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 227, "usd": 0.000141 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_4c04fd872a46", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1303, "completion_tokens": 142, "usd": 0.00011226 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a5beee38284e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 149, "usd": 0.00011424 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37245d26f903", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1509, "completion_tokens": 126, "usd": 0.00012078 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_4e2b952bb502", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1447, "completion_tokens": 210, "usd": 0.00013722 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_803282d49f8b", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 123, "usd": 0.0001035 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_c5f95c50e3d5", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1223, "completion_tokens": 150, "usd": 0.00010938 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_8a5a0ea0ed7f", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 178, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_812811600cab", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 124, "usd": 0.00010374 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_eb78878a2ed4", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1269, "completion_tokens": 132, "usd": 0.00010782 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2855, "completion_tokens": 273, "usd": 0.0011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0932e89999f7", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 142, "usd": 0.00010968 }, { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 2684, "completion_tokens": 126, "usd": 0.0009186 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2682, "completion_tokens": 4855, "usd": 0.017247 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_154346b1b35e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 189, "usd": 0.00012174 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_388513c72ff0", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1361, "completion_tokens": 186, "usd": 0.0001263 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2864, "completion_tokens": 315, "usd": 0.0011427 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_1d8ff02cb8d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 104, "usd": 0.00010062 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2603, "completion_tokens": 292, "usd": 0.0010437 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.6, "prompt_tokens": 2522, "completion_tokens": 3596, "usd": 0.01331 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_43b809e3abdd", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 111, "usd": 0.00009636 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1524, "completion_tokens": 209, "usd": 0.0006453 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_62160dffa87e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 167, "usd": 0.00011028 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f2e5587a73ca", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 119, "usd": 0.00009984 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1567, "completion_tokens": 220, "usd": 0.0006681 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00254118, "super": 0.0076527, "ultra": 0.039192 }, "counterfactual_ultra_usd": 0.103653, "spent_usd_by_tier": { "nano": 0.00292848, "super": 0.0076527, "ultra": 0.039673 }, "prompt_tokens_by_tier": { "nano": 32888, "super": 402911, "ultra": 9229 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 91.54, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-2", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_ade93ba244fa", "verdict_id": "verdict_ade93ba244fa", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The skill diff adds five lines to SKILL.md with no deletions. Eleven replays passed, but two were held for human review. In \"Triage acme/search-indexer\" step t003, the replay labeled issue #502 as 'enhancement' while the golden label is 'question'. In \"Compute 17% of 240\" step m001, the output differed with no clear cause in the diff, suggesting nondeterminism. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5da690453eaf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 101, "usd": 0.0000933 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_66be25a1b831", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1206, "completion_tokens": 186, "usd": 0.000117 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_9803f74fad28", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay mislabeling of issue #502 differs: golden labels it 'question', replay labels it 'enhancement'. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1406, "completion_tokens": 137, "usd": 0.00011724 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3201, "completion_tokens": 306, "usd": 0.0012357 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3206, "completion_tokens": 1239, "usd": 0.006923 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_287402e48fdd", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1337, "completion_tokens": 114, "usd": 0.00010758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ad012c68b2ce", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1290, "completion_tokens": 113, "usd": 0.00010452 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_c31ee7fd6635", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1402, "completion_tokens": 185, "usd": 0.00012852 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3138, "completion_tokens": 298, "usd": 0.0012096 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3102, "completion_tokens": 3988, "usd": 0.015066 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_a3e558951509", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1430, "completion_tokens": 194, "usd": 0.00013236 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3010, "completion_tokens": 304, "usd": 0.0011766 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7af9bb18724a", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1345, "completion_tokens": 187, "usd": 0.00012558 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9ea5f7e643be", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 161, "usd": 0.00012204 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_acc745f38e82", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 131, "usd": 0.00011616 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3ac050839ba", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 98, "usd": 0.0000918 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1521, "completion_tokens": 209, "usd": 0.0006444 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1e223b9eccf6", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 68, "usd": 0.00008652 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1573, "completion_tokens": 219, "usd": 0.000669 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1456, "completion_tokens": 1580, "usd": 0.006196 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a03ddffaa752", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 102, "usd": 0.00009714 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00143976, "super": 0.0049353, "ultra": 0.028185 }, "counterfactual_ultra_usd": 0.066855, "spent_usd_by_tier": { "nano": 0.00176964, "super": 0.0049353, "ultra": 0.028944 }, "prompt_tokens_by_tier": { "nano": 20814, "super": 251126, "ultra": 8109 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 91.996, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-2", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_53ec017dd4da", "verdict_id": "verdict_53ec017dd4da", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 6 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 6 of 13 replays. The system compared the edited skill against 13 recorded cases and found 6 regressions where the agent attempted new github.comment calls that the sandbox rejected, all occurring at step t005 or t006. The other 7 cases passed without behavioral change. The edit modified SKILL.md with one line added and one removed. Cost was negligible. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_aeb36d32b26b", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 121, "usd": 0.00009738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_fb3621bc1546", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 145, "usd": 0.0001071 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_a026774bb5dc", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1348, "completion_tokens": 140, "usd": 0.00011448 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_082c7d58917e", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_e82538fe8409", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_4281d13e3b01", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7ee8c01d51f9", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4f8712b7b45d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_650cb16fa7da", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_6f78304e2ae4", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_96b50023c3c8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_575a1d924d31", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 154, "usd": 0.00010716 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8419e2ca5496", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 121, "usd": 0.00010098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0006201, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.009554, "spent_usd_by_tier": { "nano": 0.00133176, "super": 0.0, "ultra": 0.000818 }, "prompt_tokens_by_tier": { "nano": 15540, "super": 419165, "ultra": 518 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 126.098, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-1", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_b02f50bd7d3c", "verdict_id": "verdict_b02f50bd7d3c", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The edit changed one line in github-triage SKILL.md. Replays showed 11 passes, 2 uncertain, and no regressions or drift. In acme/data-pipeline the model met all criteria but was held for human review at step t001. In acme/billing-api the replay counted 4 open issues against 5 in the golden run though both identified the same labeled issues; the difference is a model counting error in the final summary. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_bd63d6f067f1", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 99, "usd": 0.00009066 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f6019126d256", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1181, "completion_tokens": 165, "usd": 0.00011046 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_dd87261b8ba6", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 182, "usd": 0.00012672 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_2d58c0602c67", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1368, "completion_tokens": 182, "usd": 0.00012576 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c9de1093aa99", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1307, "completion_tokens": 130, "usd": 0.00010962 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5101ae1012ad", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay run meets criteria 1-5: it states 4 open issues (m004), lists each with identifier and type (m004), mentions #406 already labeled (m004), states no details requested (m004), and shows github.label artifacts changed (t002, t003, t…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 184, "usd": 0.00012858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3404, "completion_tokens": 279, "usd": 0.0012723 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3342, "completion_tokens": 4389, "usd": 0.016509 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_47ce9e153501", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1393, "completion_tokens": 178, "usd": 0.0001263 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3271, "completion_tokens": 303, "usd": 0.001254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4db894881c54", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1322, "completion_tokens": 115, "usd": 0.00010692 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_c8c4828aae44", "task_id": "tri-billing-api", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run reports 4 open issues while the golden run reports 5, but both runs show the same labeled issues (#31, #33, #35, #36) and both note #37 as already labeled; the discrepancy is a model counting error in the final summary. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1409, "completion_tokens": 136, "usd": 0.00011718 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3536, "completion_tokens": 339, "usd": 0.0013659 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3480, "completion_tokens": 3128, "usd": 0.012864 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_b3c44e403f10", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 153, "usd": 0.00012072 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f3095a10d7d9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 106, "usd": 0.00009444 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1613, "completion_tokens": 223, "usd": 0.0006846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_4519fd683ef9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1629, "completion_tokens": 219, "usd": 0.0006858 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1485, "completion_tokens": 1253, "usd": 0.005244 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_72d1ef164d37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 111, "usd": 0.00009858 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00144198, "super": 0.0052626, "ultra": 0.034617 }, "counterfactual_ultra_usd": 0.074379, "spent_usd_by_tier": { "nano": 0.00179826, "super": 0.0052626, "ultra": 0.035457 }, "prompt_tokens_by_tier": { "nano": 20699, "super": 251804, "ultra": 8745 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 92.158, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_e141969737af", "verdict_id": "verdict_e141969737af", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The change adds table and bullet-point parsing to the skill. Twelve test pairs passed and one pair, \"Convert at a weekend date,\" was marked uncertain because the diff does not touch date handling yet Tavily detected a rate shift from 1.0768 to 1.0800. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e5b902ded754", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 127, "usd": 0.0001092 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_8c4cb14a441e", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only modifies how items are parsed from the task (adding support for tables and bullet points), which does not affect date handling or output formatting. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 152, "usd": 0.00011688 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2627, "completion_tokens": 306, "usd": 0.0010635 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2663, "completion_tokens": 2405, "usd": 0.009878 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_3f28b051e158", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1328, "completion_tokens": 156, "usd": 0.00011712 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_bdb089b8bdbb", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 89, "usd": 0.00010212 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_2393bfd5f040", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1393, "completion_tokens": 229, "usd": 0.00013854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a029bb9d7e18", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1371, "completion_tokens": 198, "usd": 0.00012978 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_148da7df7bf5", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 135, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_c24c16f3b5fa", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 206, "usd": 0.0001293 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_b4e1ea78a932", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 124, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_cd2b69e99407", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 134, "usd": 0.00011688 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_83397ea2c263", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 96, "usd": 0.00009132 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1586, "completion_tokens": 220, "usd": 0.0006738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_37f7a8b8bf4c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1614, "completion_tokens": 229, "usd": 0.0006903 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1536, "completion_tokens": 1058, "usd": 0.00471 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8108d03dbe59", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 181, "usd": 0.000114 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1629, "completion_tokens": 216, "usd": 0.0006831 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00148758, "super": 0.0031107, "ultra": 0.014588 }, "counterfactual_ultra_usd": 0.047851, "spent_usd_by_tier": { "nano": 0.00173622, "super": 0.0033006, "ultra": 0.015195 }, "prompt_tokens_by_tier": { "nano": 20373, "super": 165893, "ultra": 4530 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 67.892, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-1", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_e62126e830f3", "verdict_id": "verdict_e62126e830f3", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: github-triage still handles all 13 past tasks.", "narrative": "Passed: github-triage still handles all 13 past tasks. The edit changed one line in SKILL.md and the regression suite shows 13 passing pairs with zero regressions, drift, or uncertain results across all model tiers. Cost remains negligible at fractions of a cent per run. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_36523cbf4c92", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1127, "completion_tokens": 137, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e70cb39d820f", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 139, "usd": 0.00010566 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_93bcdbcd3914", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 110, "usd": 0.0001077 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ba25466de412", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1385, "completion_tokens": 127, "usd": 0.00011358 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_eb462443ad9b", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1295, "completion_tokens": 110, "usd": 0.0001041 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5d5629ab1937", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1405, "completion_tokens": 148, "usd": 0.00011982 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 3128, "completion_tokens": 350, "usd": 0.0012534 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_37a737ab09a1", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 178, "usd": 0.00012708 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2971, "completion_tokens": 294, "usd": 0.0011559 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a8f42769e490", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1329, "completion_tokens": 192, "usd": 0.00012582 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dbbba87c46a3", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1401, "completion_tokens": 156, "usd": 0.0001215 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_84d120ce3156", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 283, "usd": 0.00015192 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f1c02a8e442b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_227ef3c2f411", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1560, "completion_tokens": 221, "usd": 0.0006669 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1432, "completion_tokens": 1565, "usd": 0.006127 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_ed4fe7d3e21d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 118, "usd": 0.00009954 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014565, "super": 0.0030762, "ultra": 0.006127 }, "counterfactual_ultra_usd": 0.038788, "spent_usd_by_tier": { "nano": 0.00154326, "super": 0.0030762, "ultra": 0.006568 }, "prompt_tokens_by_tier": { "nano": 17961, "super": 204884, "ultra": 1678 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 68.992, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-2", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_bcf11b31c466", "verdict_id": "verdict_bcf11b31c466", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 22 replays. Review before merging. The change modified expense-report SKILL.md with three additions and three deletions. Automated evaluation ran 22 test pairs: 21 passed, 1 was marked uncertain, and there were no regressions, drift, or improvements. The uncertain case involved filing a bakery receipt alongside a missing one; the replay output appeared to meet all six success criteria including merchant details, totals, VAT, and the missing receipt note, but the judge could not confirm correctness. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_efb6b4383a27", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1149, "completion_tokens": 164, "usd": 0.0001083 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1862, "completion_tokens": 175, "usd": 0.0007161 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_26a650f173a0", "task_id": "exp-missing-receipt", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output meets all six success criteria: it states merchant/date/total/VAT for the filed receipt, states the second receipt was not found (via the Note line), states grand total, states number of receipts filed, and states the shee…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 192, "usd": 0.00013182 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3412, "completion_tokens": 310, "usd": 0.0013026 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3379, "completion_tokens": 2251, "usd": 0.010132 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_90c84e6db9bc", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1553, "completion_tokens": 133, "usd": 0.0001251 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3978, "completion_tokens": 429, "usd": 0.0015795 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3948, "completion_tokens": 2475, "usd": 0.011373 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_9f99d91e21f6", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1301, "completion_tokens": 179, "usd": 0.00012102 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_29cce0ac5de3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1441, "completion_tokens": 141, "usd": 0.0001203 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_21bf7c25b237", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 145, "usd": 0.00012132 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_256b30ee0efe", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1303, "completion_tokens": 152, "usd": 0.00011466 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_dfea8f3d0de4", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1272, "completion_tokens": 154, "usd": 0.00011328 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_f1ddb4f3653c", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1497, "completion_tokens": 125, "usd": 0.00011982 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_ef0acc9e5a05", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 111, "usd": 0.00011418 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_11286bfe394d", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1185, "completion_tokens": 133, "usd": 0.00010302 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_325c7d095b64", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1256, "completion_tokens": 166, "usd": 0.0001152 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3011, "completion_tokens": 288, "usd": 0.0011625 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_2a529632a34a", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 142, "usd": 0.00010878 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_908da659aa18", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1203, "completion_tokens": 183, "usd": 0.0001161 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3a7988db6226", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 179, "usd": 0.00012 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_3186365c17c2", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 124, "usd": 0.00010608 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2958, "completion_tokens": 302, "usd": 0.0011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_780ead4737e0", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 188, "usd": 0.00011934 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2457, "completion_tokens": 272, "usd": 0.0009819 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_0ba4c66e04ad", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 190, "usd": 0.00012366 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_58c7d9abe244", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 102, "usd": 0.0000987 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2812, "completion_tokens": 323, "usd": 0.0011343 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2699, "completion_tokens": 3038, "usd": 0.011813 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8e9bcc22b033", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 126, "usd": 0.00009852 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1746, "completion_tokens": 221, "usd": 0.0007227 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1cc2a5f894e5", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1798, "completion_tokens": 227, "usd": 0.0007437 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1679, "completion_tokens": 1228, "usd": 0.005363 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_07aa4d542d5a", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 105, "usd": 0.00009714 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0024831, "super": 0.0095025, "ultra": 0.038681 }, "counterfactual_ultra_usd": 0.108535, "spent_usd_by_tier": { "nano": 0.0029202, "super": 0.0095025, "ultra": 0.039405 }, "prompt_tokens_by_tier": { "nano": 33998, "super": 421626, "ultra": 12033 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 92.166, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_f18e07f5c83e", "verdict_id": "verdict_f18e07f5c83e", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated skill and every pair passed with zero regressions, drift, or uncertain results. The change modified only the SKILL.md documentation file with a single line edit, leaving the core conversion logic untouched. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_8c2c5e26d4e1", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 135, "usd": 0.00011256 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_2a31e3e43325", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1341, "completion_tokens": 177, "usd": 0.00012294 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_40498db1a5f2", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 156, "usd": 0.0001164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_0f0b98c0dda1", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1536, "completion_tokens": 122, "usd": 0.00012144 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3048, "completion_tokens": 333, "usd": 0.0012141 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_a341fb0948fa", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 228, "usd": 0.00013686 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_b5a453299350", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1383, "completion_tokens": 140, "usd": 0.00011658 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ea5e1042bbf8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1427, "completion_tokens": 136, "usd": 0.00011826 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ab59cb62584a", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 207, "usd": 0.00012882 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_d101d55224e4", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1416, "completion_tokens": 113, "usd": 0.00011208 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_f96f5f3cef70", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1388, "completion_tokens": 132, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_eb8645af7d9f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 161, "usd": 0.00010908 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1645, "completion_tokens": 223, "usd": 0.0006942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_710563b99aef", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1685, "completion_tokens": 224, "usd": 0.0007071 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1566, "completion_tokens": 1498, "usd": 0.00606 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb5c11b812f5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 108, "usd": 0.0000972 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00149322, "super": 0.0026154, "ultra": 0.00606 }, "counterfactual_ultra_usd": 0.037778, "spent_usd_by_tier": { "nano": 0.00157902, "super": 0.0026154, "ultra": 0.006555 }, "prompt_tokens_by_tier": { "nano": 18497, "super": 155629, "ultra": 1818 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 64.232, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_8d5e1bc1fbc7", "verdict_id": "verdict_8d5e1bc1fbc7", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The skill diff only added an exception for a 1.0000 exchange rate, which does not apply to the uncertain case where the rate is 1.0768. Tavily detected the converted value shifted from 1.0768 to 1.0800 on a weekend date. The other 12 pairs passed with no regressions or drift. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_88832e95c9b9", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 129, "usd": 0.00010968 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_33a9c6adf157", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only added an exception for rate=1.0000, which does not apply here (rate=1.0768). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 116, "usd": 0.0001068 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2685, "completion_tokens": 284, "usd": 0.0010611 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2670, "completion_tokens": 2594, "usd": 0.010452 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_07551b4c2872", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 170, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f725843c1f80", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1394, "completion_tokens": 93, "usd": 0.00010596 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_0545f282df0a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 269, "usd": 0.00014742 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9df69746323e", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 179, "usd": 0.000126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_51bbbd84f8be", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1428, "completion_tokens": 157, "usd": 0.00012336 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_32ea0f012a38", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1343, "completion_tokens": 185, "usd": 0.00012498 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_1847ee62554a", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 123, "usd": 0.00011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_08c9424f7da6", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 268, "usd": 0.00014904 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_a144e4814df4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 112, "usd": 0.00009516 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1621, "completion_tokens": 217, "usd": 0.0006816 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e625a12fe4a6", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 140, "usd": 0.0001038 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1673, "completion_tokens": 223, "usd": 0.0007026 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1612, "completion_tokens": 1466, "usd": 0.00601 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c818c2acd85b", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 134, "usd": 0.00010344 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1664, "completion_tokens": 217, "usd": 0.0006945 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00152988, "super": 0.0031398, "ultra": 0.016462 }, "counterfactual_ultra_usd": 0.050351, "spent_usd_by_tier": { "nano": 0.00178692, "super": 0.0033297, "ultra": 0.017127 }, "prompt_tokens_by_tier": { "nano": 20374, "super": 175105, "ultra": 4614 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 71.07, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_179411b4aeac", "verdict_id": "verdict_179411b4aeac", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated skill and every pair passed with zero regressions, drift, or uncertain results. The change modified only the SKILL.md file with a minimal diff of one line added and one removed. Execution cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e9f830fa642e", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1346, "completion_tokens": 206, "usd": 0.0001302 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_bbae78eada7e", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1350, "completion_tokens": 149, "usd": 0.00011676 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_518252b4621f", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1326, "completion_tokens": 180, "usd": 0.00012276 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2718, "completion_tokens": 300, "usd": 0.0010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_6bd370a9ae4d", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1392, "completion_tokens": 102, "usd": 0.000108 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_4a19b9d5037d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1391, "completion_tokens": 244, "usd": 0.00014202 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2718, "completion_tokens": 338, "usd": 0.0011196 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_659d9e0a0119", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 202, "usd": 0.00013284 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ae6602b84c3d", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1435, "completion_tokens": 184, "usd": 0.00013026 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3036, "completion_tokens": 371, "usd": 0.0012447 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_76bd7f61f20c", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1377, "completion_tokens": 157, "usd": 0.0001203 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_7b5ee5b9fe63", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1402, "completion_tokens": 123, "usd": 0.00011364 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_40cfb64bd6bb", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1420, "completion_tokens": 99, "usd": 0.00010896 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3086, "completion_tokens": 329, "usd": 0.0012219 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2c596c557862", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 99, "usd": 0.00009204 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1662, "completion_tokens": 221, "usd": 0.0006975 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_dd37987a313e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 73, "usd": 0.00008628 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1678, "completion_tokens": 258, "usd": 0.0007356 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1532, "completion_tokens": 1227, "usd": 0.005213 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_14951b257b04", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 119, "usd": 0.00009834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0015024, "super": 0.0061047, "ultra": 0.005213 }, "counterfactual_ultra_usd": 0.048665, "spent_usd_by_tier": { "nano": 0.00158916, "super": 0.0061047, "ultra": 0.005702 }, "prompt_tokens_by_tier": { "nano": 18450, "super": 220089, "ultra": 1784 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 79.842, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-2", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_62e56accb8a3", "verdict_id": "verdict_62e56accb8a3", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now omits the repository name in the format `acme/` on 1 of 13 past tasks.", "narrative": "Blocked: github-triage now omits the repository name in the format `acme/` on 1 of 13 past tasks. The evaluation ran 13 regression pairs against the edited skill and found 12 passing and 1 regression where the required repository naming format was dropped at step m001. No drift, uncertain, or improved pairs were detected. The change introduces a clear functional regression for the \"Triage without a repository\" scenario. Roll back the edit to restore the required naming behavior. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0e10ad5bac34", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the repository name in the format `acme/` is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 115, "usd": 0.00009732 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1559, "completion_tokens": 212, "usd": 0.0006585 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1498, "completion_tokens": 4044, "usd": 0.01363 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e27ff37067f7", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1217, "completion_tokens": 145, "usd": 0.00010782 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3d9dd364ab27", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 175, "usd": 0.0001236 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_570420a9b31d", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 137, "usd": 0.00011526 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_40e6b8dc80bd", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 142, "usd": 0.00011232 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3040, "completion_tokens": 273, "usd": 0.0011577 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5a38101f8f14", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1416, "completion_tokens": 116, "usd": 0.0001128 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3149, "completion_tokens": 299, "usd": 0.0012138 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3037, "completion_tokens": 3186, "usd": 0.012595 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_1f166314792c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 221, "usd": 0.00013842 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3012, "completion_tokens": 296, "usd": 0.00117 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d9f90397db42", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1305, "completion_tokens": 99, "usd": 0.00010206 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_13d086a4d0a2", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1385, "completion_tokens": 146, "usd": 0.00011814 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_24cc0a76ffb7", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1380, "completion_tokens": 139, "usd": 0.00011616 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_86211c1cba50", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 92, "usd": 0.0000918 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_70087be29148", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1559, "completion_tokens": 219, "usd": 0.0006648 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1471, "completion_tokens": 1429, "usd": 0.005758 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3b107f1aecf5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 104, "usd": 0.00009618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00141792, "super": 0.0048648, "ultra": 0.031983 }, "counterfactual_ultra_usd": 0.070128, "spent_usd_by_tier": { "nano": 0.0017472, "super": 0.0048648, "ultra": 0.03262 }, "prompt_tokens_by_tier": { "nano": 20512, "super": 230152, "ultra": 6301 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 82.434, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-2", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_b4c1f05bcb9e", "verdict_id": "verdict_b4c1f05bcb9e", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The regression occurred in \"Triage acme/infra-terraform\" where the sandbox blocked two new github.comment calls that the original run never made. One other replay, \"Triage acme/data-pipeline,\" was held as uncertain because the trace shows label artifacts changed but the output criteria were otherwise met. Eleven replays passed. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_9aae4e6186d2", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 99, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_852cf7299d83", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1181, "completion_tokens": 166, "usd": 0.0001107 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_1f4cd7269ecb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 166, "usd": 0.00012288 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d6db457362fe", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_7d366fbfea40", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 132, "usd": 0.00011082 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_bb40249ba200", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets criteria 1-5: it states 4 open issues (m004), lists each with type (m004), notes #406 already labeled (m004), states no details requested (m004), and the trace shows github.label artifacts changed (t002, t003, t004). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1396, "completion_tokens": 186, "usd": 0.0001284 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3119, "completion_tokens": 330, "usd": 0.0012327 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3068, "completion_tokens": 4026, "usd": 0.015146 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_5cb5a86b5b0c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 171, "usd": 0.0001239 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3148, "completion_tokens": 260, "usd": 0.0011784 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_8e097e2bc153", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1341, "completion_tokens": 99, "usd": 0.00010422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dc72a3a720da", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1366, "completion_tokens": 177, "usd": 0.00012444 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d4e29e63139d", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1392, "completion_tokens": 117, "usd": 0.0001116 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1bcf329ed33b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_20de0884c5fb", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 80, "usd": 0.00008868 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1558, "completion_tokens": 219, "usd": 0.0006645 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1436, "completion_tokens": 1347, "usd": 0.005477 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b123eb9839bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00130332, "super": 0.0030756, "ultra": 0.020623 }, "counterfactual_ultra_usd": 0.051024, "spent_usd_by_tier": { "nano": 0.00163266, "super": 0.0030756, "ultra": 0.021306 }, "prompt_tokens_by_tier": { "nano": 19319, "super": 270304, "ultra": 4905 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 98.393, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_dfb7b744c7ec", "verdict_id": "verdict_dfb7b744c7ec", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The skill diff only adds an HTML comment with no logic changes. Twelve replays passed, but one replay for a weekend date showed the converted value moving from 1.0768 to 1.0800. The judges marked this uncertain because the diff cannot explain the shift. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_4e739b0fed3d", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1324, "completion_tokens": 133, "usd": 0.00011136 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_d7ede738861c", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds an HTML comment () with no change to steps, tools, arguments, or rules. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 103, "usd": 0.00010512 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2569, "completion_tokens": 283, "usd": 0.0010254 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2469, "completion_tokens": 2038, "usd": 0.008583 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_7f573055a9ef", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 179, "usd": 0.00012192 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f4c1c98c1a21", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 206, "usd": 0.00013236 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_6b7462d13be4", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 235, "usd": 0.00013926 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_914ca7824ded", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 139, "usd": 0.0001164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_da9dc2ce34c7", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1416, "completion_tokens": 135, "usd": 0.00011736 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_70ade45e8144", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1343, "completion_tokens": 232, "usd": 0.00013626 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_0b662be246d7", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 118, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_ca8b25ae83a2", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 137, "usd": 0.00011688 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_0bda149accdc", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 107, "usd": 0.00009468 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1482, "completion_tokens": 211, "usd": 0.0006345 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e0d76dcf56dd", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 68, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1522, "completion_tokens": 221, "usd": 0.0006555 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1347, "completion_tokens": 1626, "usd": 0.006225 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f3b045e8f40e", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 129, "usd": 0.00010218 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014943, "super": 0.0023154, "ultra": 0.014808 }, "counterfactual_ultra_usd": 0.04551, "spent_usd_by_tier": { "nano": 0.00174078, "super": 0.0025053, "ultra": 0.015434 }, "prompt_tokens_by_tier": { "nano": 20397, "super": 168576, "ultra": 4151 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 79.382, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-1", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_cbdda0881101", "verdict_id": "verdict_cbdda0881101", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now adds an extra labeled-issue line for already-labeled issue on 1 of 13 past tasks.", "narrative": "Blocked: github-triage now adds an extra labeled-issue line for already-labeled issue on 1 of 13 past tasks. The edit changed SKILL.md and was tested against 13 prior tasks, resulting in 10 passes, 1 regression, and 2 uncertain outcomes. The regression on acme/search-indexer shows the agent now includes an already-labeled issue in the newly-labeled output section, violating the required format. The two uncertain cases involve extra output lines on acme/status-page and likely nondeterminism on acme/mobile-app. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_fc1895f58dcf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1127, "completion_tokens": 99, "usd": 0.00009138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_99daafbd39ef", "task_id": "tri-status-page", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output includes extra lines '#8: bug' and '#11: enhancement' that violate the 'no additional text' success criterion. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1240, "completion_tokens": 128, "usd": 0.00010512 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2658, "completion_tokens": 264, "usd": 0.001035 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2620, "completion_tokens": 4688, "usd": 0.016684 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_204e00dc72de", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output includes an extra line '#503: bug (already labeled)' in the labeled-issues section, which violates the skill's output format: only issues newly labeled by the triage should appear there, while already-labeled issues belong… is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1341, "completion_tokens": 91, "usd": 0.0001023 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3296, "completion_tokens": 309, "usd": 0.0012669 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 6575, "completion_tokens": 5451, "usd": 0.022928 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d8383ef71ebb", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1368, "completion_tokens": 135, "usd": 0.00011448 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3281, "completion_tokens": 291, "usd": 0.0012462 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_f644709d32f1", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1304, "completion_tokens": 189, "usd": 0.0001236 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3046, "completion_tokens": 253, "usd": 0.0011415 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_92e4b2c50b1c", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1383, "completion_tokens": 93, "usd": 0.0001053 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3380, "completion_tokens": 427, "usd": 0.0013983 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_574e79e6ac2e", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1404, "completion_tokens": 188, "usd": 0.00012936 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3033, "completion_tokens": 295, "usd": 0.0011754 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7213bfe2b9f8", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1331, "completion_tokens": 113, "usd": 0.00010698 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3072, "completion_tokens": 316, "usd": 0.001206 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 6265, "completion_tokens": 5504, "usd": 0.022777000000000002 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_a05f5333a824", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1401, "completion_tokens": 178, "usd": 0.00012678 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3677, "completion_tokens": 292, "usd": 0.0013659 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_1891e80878db", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1400, "completion_tokens": 157, "usd": 0.00012168 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3218, "completion_tokens": 360, "usd": 0.0012894 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d5662e056952", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 97, "usd": 0.00009156 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1517, "completion_tokens": 221, "usd": 0.000654 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e3d06ab590fa", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 116, "usd": 0.00009732 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_18d4cf624d2d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 82, "usd": 0.00008946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00140532, "super": 0.0117786, "ultra": 0.062389 }, "counterfactual_ultra_usd": 0.123407, "spent_usd_by_tier": { "nano": 0.00172998, "super": 0.0117786, "ultra": 0.063177 }, "prompt_tokens_by_tier": { "nano": 20669, "super": 284119, "ultra": 15885 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 111.593, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-expense-report-2", "skill": "expense-report", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_5b02579ddb87", "verdict_id": "verdict_5b02579ddb87", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 22 replays. Review before merging. The diff only renames a parameter in expense-report from 'file' to 'filename'. Two uncertain replays involve that skill but show identical tool calls and outcomes before and after the change. The third uncertain replay cleans an unrelated CSV file. All 19 passing replays confirm existing behavior works. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2f14f21d6d9f", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1106, "completion_tokens": 152, "usd": 0.00010284 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_e92a7282295d", "task_id": "exp-missing-receipt", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only changes the parameter name for receipts.parse from 'file' to 'filename', but both runs call the tool with 'file' and both successfully parse the first receipt and fail to find the second. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 142, "usd": 0.00011904 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3174, "completion_tokens": 314, "usd": 0.0012348 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3103, "completion_tokens": 1865, "usd": 0.008698 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_a6b930ea6aff", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1654, "completion_tokens": 102, "usd": 0.00012372 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 4021, "completion_tokens": 373, "usd": 0.001542 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3948, "completion_tokens": 1847, "usd": 0.009489 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_e2914d406490", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1289, "completion_tokens": 140, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_07a85852522a", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 142, "usd": 0.00011982 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_85a67c3ff72c", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1454, "completion_tokens": 112, "usd": 0.00011412 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_afcfe0768c5a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1291, "completion_tokens": 177, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a64afbf6b2cb", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1272, "completion_tokens": 138, "usd": 0.00010944 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_9c6ec62f2eb9", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 172, "usd": 0.00013254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6ad0a889dbc3", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1471, "completion_tokens": 158, "usd": 0.00012618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_b6e697e319df", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 161, "usd": 0.00011262 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_8ffffbf26485", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1256, "completion_tokens": 157, "usd": 0.00011304 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2885, "completion_tokens": 317, "usd": 0.0011508 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_56a48c4930ba", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 164, "usd": 0.00011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_4c8e63150ce8", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 175, "usd": 0.0001176 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2672, "completion_tokens": 323, "usd": 0.0010923 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_aa8e42f0327e", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay writes the correct cleaned file (step t007) with 3 data rows, meeting all success criteria for the file content. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1373, "completion_tokens": 94, "usd": 0.00010494 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3375, "completion_tokens": 312, "usd": 0.0012933 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.7, "prompt_tokens": 3293, "completion_tokens": 4581, "usd": 0.017036 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_c3600f8c1037", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 155, "usd": 0.0001128 }, { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 2841, "completion_tokens": 65, "usd": 0.0009108 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2735, "completion_tokens": 3489, "usd": 0.013202 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_64e31b08b933", "task_id": "csv-header-only", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff changes a parameter name in expense-report/SKILL.md (file → filename), but the task is cleaning leads.csv, which is unrelated to the expense-report skill. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 275, "usd": 0.00014334 }, { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 3010, "completion_tokens": 70, "usd": 0.000966 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3014, "completion_tokens": 1665, "usd": 0.008009 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_c2e995113d65", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 171, "usd": 0.00011838 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_18bb09c7d462", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1225, "completion_tokens": 102, "usd": 0.00009798 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2771, "completion_tokens": 344, "usd": 0.0011409 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 5964, "completion_tokens": 6040, "usd": 0.024084 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_de7f8d3695ad", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 107, "usd": 0.00009396 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1620, "completion_tokens": 223, "usd": 0.0006867 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_40e8a755ae4f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1672, "completion_tokens": 223, "usd": 0.0007023 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1552, "completion_tokens": 1027, "usd": 0.004633 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b58e85552abd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00248568, "super": 0.0107199, "ultra": 0.085151 }, "counterfactual_ultra_usd": 0.159162, "spent_usd_by_tier": { "nano": 0.00328212, "super": 0.0107199, "ultra": 0.085876 }, "prompt_tokens_by_tier": { "nano": 38122, "super": 530767, "ultra": 24037 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 147.663, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-expense-report-1", "skill": "expense-report", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_9cdd938ccf18", "verdict_id": "verdict_9cdd938ccf18", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 6 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 6 of 22 replays. Review before merging. The diff only rewords the skill description from \"receipt\" to \"document\" without changing any functional steps, tools, or rules. Six test cases were held because this cosmetic change could not be verified as behaviorally neutral by the automated judges. Sixteen cases passed with no regressions or drift detected. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2cfa611323e6", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1094, "completion_tokens": 143, "usd": 0.00009996 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_f3f61876a85b", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1442, "completion_tokens": 142, "usd": 0.0001206 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2816, "completion_tokens": 306, "usd": 0.0011202 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2770, "completion_tokens": 1577, "usd": 0.007501 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_03e985bf60c4", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1589, "completion_tokens": 164, "usd": 0.0001347 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3801, "completion_tokens": 414, "usd": 0.0015129 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3809, "completion_tokens": 2692, "usd": 0.011885 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_b14c6d6a1857", "task_id": "exp-software", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description (receipt→document) without changing any functional step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1338, "completion_tokens": 184, "usd": 0.00012444 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 2634, "completion_tokens": 289, "usd": 0.0010503 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2616, "completion_tokens": 3120, "usd": 0.011976 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_758cddadc695", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1513, "completion_tokens": 153, "usd": 0.0001275 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3346, "completion_tokens": 327, "usd": 0.0012981 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5e47aca6907b", "task_id": "exp-dinner-parking", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field in SKILL.md (changing 'receipts' to 'receipts, invoices, or other financial documents' and 'receipt file' to 'file'), which does not alter any tool, argument, step, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1456, "completion_tokens": 189, "usd": 0.00013272 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3153, "completion_tokens": 316, "usd": 0.0012303 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3159, "completion_tokens": 2102, "usd": 0.009465 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a8a18a97157f", "task_id": "exp-office-supplies", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description text without changing any functional logic, tool calls, or rules. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1327, "completion_tokens": 139, "usd": 0.00011298 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2499, "completion_tokens": 273, "usd": 0.0009954 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2453, "completion_tokens": 1851, "usd": 0.008006 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_2a8fe0325d1e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1320, "completion_tokens": 141, "usd": 0.00011304 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_21460568387a", "task_id": "exp-lunch-train", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description text (adding 'invoices, or other financial documents' and changing 'receipt' to 'document'); it does not change any tool, step, argument, or logic rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1517, "completion_tokens": 145, "usd": 0.00012582 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3285, "completion_tokens": 383, "usd": 0.0013302 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3287, "completion_tokens": 2972, "usd": 0.012203 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_1b2e69a62019", "task_id": "exp-berlin-trip", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field in SKILL.md (changing 'receipt' to 'document'), which is a cosmetic documentation change that cannot plausibly alter the model's output formatting or content selection. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1441, "completion_tokens": 165, "usd": 0.00012606 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3139, "completion_tokens": 431, "usd": 0.0013296 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3140, "completion_tokens": 2065, "usd": 0.009335 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_df0530c17b4e", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 155, "usd": 0.0001119 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_5b34a19cd981", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 233, "usd": 0.00013002 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_b1ea18acb350", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 106, "usd": 0.00010014 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_6d6c20462d5a", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 135, "usd": 0.00010746 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_1d5047ec8fd2", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay's written file (t003) contains 3 data rows plus header, meeting all success criteria for the cleaned file content. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 105, "usd": 0.00010224 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2706, "completion_tokens": 280, "usd": 0.0010638 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2645, "completion_tokens": 4031, "usd": 0.014738 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_a7ca6fd2483c", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 117, "usd": 0.00010368 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2652, "completion_tokens": 300, "usd": 0.0010656 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_8ed25d9902cb", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1213, "completion_tokens": 154, "usd": 0.00010974 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2195, "completion_tokens": 268, "usd": 0.0008997 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_bdb9bd3422e8", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 210, "usd": 0.00012846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_03c42fb9fb74", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 131, "usd": 0.00010638 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2610, "completion_tokens": 291, "usd": 0.0010449 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2538, "completion_tokens": 3216, "usd": 0.012186 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d54e294c1ad8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 100, "usd": 0.00009372 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3cb515f6830e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 116, "usd": 0.00009804 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1629, "completion_tokens": 205, "usd": 0.0006732 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c51d156486d2", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 97, "usd": 0.0000945 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0025041, "super": 0.0146142, "ultra": 0.097295 }, "counterfactual_ultra_usd": 0.18452, "spent_usd_by_tier": { "nano": 0.00359682, "super": 0.0146142, "ultra": 0.098182 }, "prompt_tokens_by_tier": { "nano": 41443, "super": 525342, "ultra": 27028 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 134.543, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-1", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_4eb69b215c6b", "verdict_id": "verdict_4eb69b215c6b", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The edit updated SKILL.md with two additions and zero deletions. The system re-ran the full regression suite of 22 pairs covering the skill's behavior. Every pair produced the expected result with no regressions, drift, or uncertain outcomes. The change is purely additive documentation or configuration that does not alter execution logic. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_9797515450fd", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1070, "completion_tokens": 110, "usd": 0.0000906 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_562f1bacbb4a", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1405, "completion_tokens": 134, "usd": 0.00011646 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3186, "completion_tokens": 269, "usd": 0.0011979 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_118b690af2c2", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1670, "completion_tokens": 121, "usd": 0.00012924 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3822, "completion_tokens": 419, "usd": 0.0015237 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3733, "completion_tokens": 1548, "usd": 0.008377 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_17c39cd8a28a", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1313, "completion_tokens": 191, "usd": 0.00012462 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_89a8beb01ad9", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 212, "usd": 0.00013662 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5a08a4d190cc", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1466, "completion_tokens": 182, "usd": 0.00013164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a04984da2f2e", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1315, "completion_tokens": 136, "usd": 0.00011154 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_6c284a0936b6", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1332, "completion_tokens": 144, "usd": 0.00011448 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37dcfb667a2b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1533, "completion_tokens": 133, "usd": 0.0001239 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_f63fa53e365f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1471, "completion_tokens": 111, "usd": 0.0001149 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_596660827ddb", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 173, "usd": 0.00011622 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_35dd5c46b4f3", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1259, "completion_tokens": 201, "usd": 0.00012378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_24bcb89d6737", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 142, "usd": 0.00010806 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_3053a2b19841", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 197, "usd": 0.00012162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_df65d3ad0120", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 193, "usd": 0.00012264 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_175a9b556a3d", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 273, "usd": 0.00013968 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2640, "completion_tokens": 330, "usd": 0.001089 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_6396f7029b97", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1358, "completion_tokens": 166, "usd": 0.00012132 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2464, "completion_tokens": 239, "usd": 0.0009543 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2380, "completion_tokens": 3240, "usd": 0.0121 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_7ff7643575cd", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 196, "usd": 0.00012438 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_07fbc9fb046d", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 164, "usd": 0.0001143 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2572, "completion_tokens": 313, "usd": 0.0010533 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2578, "completion_tokens": 2701, "usd": 0.010681 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_11e6e85db7b9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 105, "usd": 0.0000942 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1473, "completion_tokens": 223, "usd": 0.0006426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_6781c847f496", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1513, "completion_tokens": 219, "usd": 0.000651 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1413, "completion_tokens": 1628, "usd": 0.006297 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_16136c6d8859", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 97, "usd": 0.00009522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00256146, "super": 0.0071118, "ultra": 0.037455 }, "counterfactual_ultra_usd": 0.100399, "spent_usd_by_tier": { "nano": 0.00339522, "super": 0.0071118, "ultra": 0.037972 }, "prompt_tokens_by_tier": { "nano": 37571, "super": 423997, "ultra": 10345 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 98.063, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-1", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_5b9237adf376", "verdict_id": "verdict_5b9237adf376", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The system replayed 13 test cases against the updated csv-cleaner skill; 11 passed but two cases, cleaning support/tickets.csv and signups.csv, produced uncertain results where the judges could not verify the changes against the skill diff. The cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_95b0aeaae315", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 222, "usd": 0.00012966 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ad0ef0f6f262", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1227, "completion_tokens": 136, "usd": 0.00010626 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_4bc22f69c2f4", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1257, "completion_tokens": 123, "usd": 0.00010494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_341f1aafe06d", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 107, "usd": 0.00009984 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2681, "completion_tokens": 280, "usd": 0.0010563 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2579, "completion_tokens": 1634, "usd": 0.007481 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_89cbeb5f1106", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 184, "usd": 0.00011976 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2609, "completion_tokens": 286, "usd": 0.0010401 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_638b93eff172", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 212, "usd": 0.00012822 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_57f1a1c928ec", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1247, "completion_tokens": 142, "usd": 0.0001089 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_ed700d3411da", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay meets all success criteria: signups.clean.csv has 3 rows, no empty ids, no duplicates, and the original file has 4 rows. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 188, "usd": 0.0001215 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2507, "completion_tokens": 298, "usd": 0.0010203 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 5294, "completion_tokens": 6386, "usd": 0.024451999999999998 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_f83b9f46b05b", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 108, "usd": 0.00010218 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7323ba2c2619", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 106, "usd": 0.00010014 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2e08956d5846", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 118, "usd": 0.00009876 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_8ad7f2f013aa", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1517, "completion_tokens": 219, "usd": 0.0006522 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 1237, "usd": 0.00511 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3f0ad4c092bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 103, "usd": 0.0000945 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00140142, "super": 0.0037689, "ultra": 0.037043 }, "counterfactual_ultra_usd": 0.071142, "spent_usd_by_tier": { "nano": 0.0017292, "super": 0.0037689, "ultra": 0.037657 }, "prompt_tokens_by_tier": { "nano": 19764, "super": 232962, "ultra": 9616 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 194.078, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-github-triage-1", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_4bc9c9ce0479", "verdict_id": "verdict_4bc9c9ce0479", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The change only adds a comment line stating the steps are unchanged and modifies no logic, rules, or tool calls. Twelve replays passed and one was marked uncertain because the judge could not verify the intent of the comment-only edit. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_c6a8e1abb734", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 149, "usd": 0.00010554 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_4e23436145b0", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 145, "usd": 0.0001071 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e1dd839734fb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 164, "usd": 0.0001224 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_860252f0aeb1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1380, "completion_tokens": 125, "usd": 0.0001128 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac0076e8ac87", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1307, "completion_tokens": 146, "usd": 0.00011346 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_0247b5972b80", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1408, "completion_tokens": 167, "usd": 0.00012456 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3092, "completion_tokens": 316, "usd": 0.001212 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2988, "completion_tokens": 2882, "usd": 0.011634 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_07bb9b326000", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1478, "completion_tokens": 199, "usd": 0.00013644 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3109, "completion_tokens": 347, "usd": 0.001245 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_ef274337a616", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a comment line stating 'the steps are unchanged' and does not modify any logic, rules, or tool calls. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 113, "usd": 0.00010752 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2982, "completion_tokens": 298, "usd": 0.0011628 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2894, "completion_tokens": 1567, "usd": 0.007595 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_ee6719208401", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1409, "completion_tokens": 146, "usd": 0.00011958 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_44c02d5b3570", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1389, "completion_tokens": 134, "usd": 0.0001155 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3125, "completion_tokens": 349, "usd": 0.0012516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_bfdc8aa6fc39", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 100, "usd": 0.00009228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a63316f74650", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 72, "usd": 0.00008532 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1515, "completion_tokens": 219, "usd": 0.0006516 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1395, "completion_tokens": 1345, "usd": 0.00543 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_86b52a6672e1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1175, "completion_tokens": 121, "usd": 0.00009954 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00144204, "super": 0.005523, "ultra": 0.024659 }, "counterfactual_ultra_usd": 0.065322, "spent_usd_by_tier": { "nano": 0.00178398, "super": 0.005523, "ultra": 0.025211 }, "prompt_tokens_by_tier": { "nano": 20773, "super": 251220, "ultra": 7580 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 196.872, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_2abff9535186", "verdict_id": "verdict_2abff9535186", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 1 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 1 changed because the world did, not the skill. The regression suite ran 13 pairs with 12 passing and 1 drift. The drift occurred in \"Convert London costs from GBP\" where Tavily detected the GBP/USD exchange rate moved from 1.2702 to 1.3105 on 2026-10-13, causing the output to update correctly. The skill logic remains sound. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_3e8797c16149", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 110, "usd": 0.00010656 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_608d63d15c36", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 162, "usd": 0.00012 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_229dc213dbd6", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 160, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_e71800e8113a", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1406, "completion_tokens": 104, "usd": 0.00010932 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_16ce4eaa1d1a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 231, "usd": 0.00013758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_bd178338e4b1", "task_id": "fx-london", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The GBP/USD exchange rate on 2026-10-13 was 1.2702 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 197, "usd": 0.0001317 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2584, "completion_tokens": 286, "usd": 0.0010326 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2574, "completion_tokens": 1593, "usd": 0.007353 } ], "control_outcome": null, "drift": [ { "fact_id": "t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.2702", "new_value": "1.3105" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_f6108436cd3f", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1451, "completion_tokens": 172, "usd": 0.00012834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_133f03cc5707", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 209, "usd": 0.00013146 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_cb6d6a6fa18e", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 93, "usd": 0.00010872 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_9bc309eb22db", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1377, "completion_tokens": 170, "usd": 0.00012342 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_c0d90d4cb42f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 93, "usd": 0.0000906 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_7d778bb3b477", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 68, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1563, "completion_tokens": 221, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1412, "completion_tokens": 1979, "usd": 0.007349 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9620dc2201ff", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00148176, "super": 0.0017004, "ultra": 0.014702 }, "counterfactual_ultra_usd": 0.043215, "spent_usd_by_tier": { "nano": 0.00161928, "super": 0.0018969, "ultra": 0.015377 }, "prompt_tokens_by_tier": { "nano": 19152, "super": 153495, "ultra": 4322 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 81.315, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_d0ecbd35ba51", "verdict_id": "verdict_d0ecbd35ba51", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 5 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 5 changed because the world did, not the skill. The run covered 8 passing cases and 5 drift cases with no regressions or improvements. All five drift cases stem from the EUR/USD exchange rate on 2026-10-14 shifting from 1.0800 to 1.163, verified by Tavily. The skill logic remains correct; only the external market data moved. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_24416e3f8916", "task_id": "fx-no-date", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1347, "completion_tokens": 155, "usd": 0.00011802 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2378, "completion_tokens": 353, "usd": 0.0010311 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2368, "completion_tokens": 1378, "usd": 0.006502 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_c04a94e117fd", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "drift", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 160, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_823b4ddae43a", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 157, "usd": 0.00011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_9326a185dca9", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1402, "completion_tokens": 97, "usd": 0.0001074 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2585, "completion_tokens": 287, "usd": 0.0010338 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_48d13ccbd36d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 228, "usd": 0.00013686 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_24d29ab3083c", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1407, "completion_tokens": 141, "usd": 0.00011826 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_d540794f1c98", "task_id": "fx-coworking-courier", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1426, "completion_tokens": 152, "usd": 0.00012204 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2595, "completion_tokens": 407, "usd": 0.0011448 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2602, "completion_tokens": 1322, "usd": 0.006568 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ecc9e77802a1", "task_id": "fx-software", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "finance.rate returned 1.08 on 2026-10-14 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1366, "completion_tokens": 170, "usd": 0.00012276 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2410, "completion_tokens": 343, "usd": 0.0010317 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2437, "completion_tokens": 1339, "usd": 0.006454 } ], "control_outcome": null, "drift": [ { "fact_id": "obs_t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.08", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_ce1c4a74b4df", "task_id": "fx-hamburg-conference", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1439, "completion_tokens": 172, "usd": 0.00012762 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2862, "completion_tokens": 327, "usd": 0.0011529 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2930, "completion_tokens": 1437, "usd": 0.007241 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_84352baa167a", "task_id": "fx-berlin-trip", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 177, "usd": 0.00012786 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2602, "completion_tokens": 383, "usd": 0.0011253 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2649, "completion_tokens": 1367, "usd": 0.00675 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_68a265f4d0b3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "drift", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 101, "usd": 0.00009468 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1523, "completion_tokens": 217, "usd": 0.0006522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_eac0515b4a0c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1563, "completion_tokens": 221, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1417, "completion_tokens": 1693, "usd": 0.006496 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_aa0845d7fbd7", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 84, "usd": 0.0000921 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014898, "super": 0.0078396, "ultra": 0.040011 }, "counterfactual_ultra_usd": 0.089107, "spent_usd_by_tier": { "nano": 0.00191844, "super": 0.0087852, "ultra": 0.04097 }, "prompt_tokens_by_tier": { "nano": 22702, "super": 174832, "ultra": 15041 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 92.601, "simulated_models": false, "repeat": 0, "error": null } ] }, "nano_only": { "config": { "name": "nano_only", "description": "Nano only — every pair judged by nano alone (Level 0 and drift as in the cascade).", "experiments": [ "E4" ], "mode": "edits", "cascade": { "mode": "nano_only", "level0": true, "drift": true, "controls": true }, "single_judge_blocks": true, "types": [ "R1", "R2", "R3", "R4", "R5", "R6", "N0", "D1" ], "noise_repeats": 3, "runner": "local", "concurrency": 2, "reuse_replays": "cascade", "max_traces": 40 }, "config_path": "bench/configs/nano_only.yaml", "command": "ratchet bench run --config bench/configs/nano_only.yaml --out bench/results/2026-10-16 --types D1 --limit 2 --yes", "started_at": "2026-10-16T12:44:09.109374Z", "updated_at": "2026-10-16T13:56:55.205408Z", "edits": [ { "bench_id": "n0-github-triage-2", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_c93e64342a3c", "verdict_id": "verdict_c93e64342a3c", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The system ran 13 evaluation pairs against the edited github-triage skill, resulting in 11 passes, 1 regression, 1 uncertain, and no drift or improvements. The regression occurred on \"Summarize a short note\" where the replay changed \"will stay unchanged\" to \"stays unchanged,\" altering the agenda criterion from a statement of constancy to a future-oriented promise. The uncertain case on \"Triage acme/data-pipeline\" was held because the judge answer was unreadable. The edit cost $0.0018. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_811b35141f2c", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 99, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_b3d50ce1b799", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1217, "completion_tokens": 135, "usd": 0.00010542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_7741c40ef280", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 154, "usd": 0.00011826 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_abe3d0283a76", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 160, "usd": 0.00012078 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_a0217d5141b7", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 142, "usd": 0.00011394 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_f5d663dbc019", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Judge answer unreadable; treated as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 4829, "completion_tokens": 900, "usd": 0.00050574 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7e7f0890e15f", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1370, "completion_tokens": 205, "usd": 0.0001314 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a1def1b764dc", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 111, "usd": 0.0001074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_0490537a6338", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 218, "usd": 0.00013572 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_8b534e552950", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 117, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_7e792837459f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 92, "usd": 0.00009036 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_95212662e369", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a24da7604ca9", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay changes 'will stay unchanged' to 'stays unchanged', which alters the meaning of the agenda criterion from a present tense statement of constancy to a future‑oriented promise, failing the requirement that the agenda is unchanged is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 132, "usd": 0.00010224 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00182106, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.027814, "spent_usd_by_tier": { "nano": 0.00182106, "super": 0.0, "ultra": 0.000761 }, "prompt_tokens_by_tier": { "nano": 20203, "super": 0, "ultra": 323 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 120.159, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-1", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_97ddc8a9c1fd", "verdict_id": "verdict_97ddc8a9c1fd", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 5 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 5 of 13 replays. Review before merging. Ratchet replayed 13 past tasks against the edit to github-triage and compared each with its blessed result. On \"Triage acme/search-indexer\": Got worse. The label for issue #502 was changed from 'question' to 'enhancement', violating the success criterion that it must be labeled as 'question' is no longer met. (step t003) On \"Triage acme/auth-service\": Got worse. The replay omits the required phrase \"Asked for details: #91\" which is part of the success criteria is no longer met. (step t005) On \"Triage acme/mobile-app\": Got worse. The replay omits the required request for details on #64 and the citation of github.comment as an artifact changed is no longer met. (step t005) Judging cost $0.0014 (nano). Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_29198b3e615a", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 121, "usd": 0.00009738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_ae5049772f22", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1175, "completion_tokens": 133, "usd": 0.00010242 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_6a4a7b2c3a20", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The label for issue #502 was changed from 'question' to 'enhancement', violating the success criterion that it must be labeled as 'question' is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1365, "completion_tokens": 127, "usd": 0.00011238 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ff7547adf9e1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1364, "completion_tokens": 122, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_95dc2c9a2266", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits the required phrase \"Asked for details: #91\" which is part of the success criteria is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1286, "completion_tokens": 141, "usd": 0.000111 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_1ac70e25ffb3", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1353, "completion_tokens": 152, "usd": 0.00011766 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_3be8109cfb2f", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 155, "usd": 0.00012036 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_650e3f7e3ede", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits the required request for details on #64 and the citation of github.comment as an artifact changed is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1308, "completion_tokens": 106, "usd": 0.00010392 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_e5bee22bc04e", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits the phrase \"Asked for details: #36\" which is a required criterion is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1400, "completion_tokens": 116, "usd": 0.00011184 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_0c922bfa825f", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits the required mention that #15 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 98, "usd": 0.00010656 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_00c8cee98bc3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 97, "usd": 0.00009156 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_05517158aa56", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_bdbe9648c85c", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 119, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00137346, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.021332, "spent_usd_by_tier": { "nano": 0.00137346, "super": 0.0, "ultra": 0.000814 }, "prompt_tokens_by_tier": { "nano": 16655, "super": 0, "ultra": 469 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 43.739, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-2", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_695f39f8a962", "verdict_id": "verdict_695f39f8a962", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 22 replays. Review before merging. The system replayed 22 golden pairs against the edited expense-report skill, resulting in 21 passes and 1 regression. The failure occurred on the \"Clean payments.csv\" case where the replay produced \"Rows in: 4\" instead of the expected \"Rows in: 5\" at step t001. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_db2fa52f33ec", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1106, "completion_tokens": 141, "usd": 0.0001002 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_663c04fc7679", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 186, "usd": 0.00013038 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_b8eacf7ca9eb", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1637, "completion_tokens": 162, "usd": 0.0001371 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_df4a9a6d2d5e", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 180, "usd": 0.00011982 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_2aba1c6b9a48", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 186, "usd": 0.00013038 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_4d8a32096183", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 229, "usd": 0.00014148 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_5d79f77bf29a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1315, "completion_tokens": 136, "usd": 0.00011154 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_fa5cfd551a14", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 141, "usd": 0.00011232 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_bcd2f6c4de1b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 146, "usd": 0.0001263 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6c9bfaed6179", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 133, "usd": 0.00011946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_979bc36816b3", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 170, "usd": 0.00011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_0444b760bf07", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 195, "usd": 0.00012306 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_997e188d6e47", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 160, "usd": 0.0001131 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_d40c1503ab73", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 163, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_8111d388d469", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 202, "usd": 0.00012552 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0727abbc4d2a", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay reports \"Rows in: 4\" whereas the golden output specifies \"Rows in: 5\" is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 284, "usd": 0.00014232 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_cfff2f4aa409", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 173, "usd": 0.00011646 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_52606712bd67", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 163, "usd": 0.00011502 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_542487f91fa5", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 180, "usd": 0.00011814 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_599436fdbbc6", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 136, "usd": 0.00010164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_de6195626ea9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1122, "completion_tokens": 68, "usd": 0.00008364 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e212cacce84", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1187, "completion_tokens": 119, "usd": 0.00009978 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00259572, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.039609, "spent_usd_by_tier": { "nano": 0.00259572, "super": 0.0, "ultra": 0.000567 }, "prompt_tokens_by_tier": { "nano": 28650, "super": 0, "ultra": 273 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 60.601, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_21b664fe580c", "verdict_id": "verdict_21b664fe580c", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system evaluated the fx-expense-converter skill against 13 test cases, finding 12 passing and 1 regression. The failure occurred on the \"Convert at a weekend date\" test where the applied exchange rate date shifted from 2026-10-11 to 2026-10-09, changing the rate value from 1.0768 to 1.0800. No drift, improvements, or uncertain results were detected. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_885b30526626", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1347, "completion_tokens": 129, "usd": 0.00011178 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_fcf301073639", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The date in the rate used is 2026-10-09 instead of the required 2026-10-11 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1378, "completion_tokens": 145, "usd": 0.00011748 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_82a70ccb2af8", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1351, "completion_tokens": 182, "usd": 0.00012474 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_973ca2670ff2", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1405, "completion_tokens": 102, "usd": 0.00010878 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3afca948a379", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 225, "usd": 0.00013896 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_f5be008e5f9d", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 198, "usd": 0.00013194 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_b55b161ede07", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1474, "completion_tokens": 93, "usd": 0.00011076 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_e70094e8923d", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1354, "completion_tokens": 194, "usd": 0.0001278 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5bde00727c59", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1475, "completion_tokens": 93, "usd": 0.00011082 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_96be56214418", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1435, "completion_tokens": 133, "usd": 0.00011802 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d1a68d58b4ae", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 157, "usd": 0.00010668 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_96c18d79778a", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 68, "usd": 0.00008652 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9cb3c3858cd1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 84, "usd": 0.0000921 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00148638, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.02297, "spent_usd_by_tier": { "nano": 0.00153642, "super": 0.0001908, "ultra": 0.00073 }, "prompt_tokens_by_tier": { "nano": 18263, "super": 585, "ultra": 325 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 18.475, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-1", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_40b8b05f498f", "verdict_id": "verdict_40b8b05f498f", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would close issues in 5 of 13 replays.", "narrative": "Blocked: the new github-triage would close issues in 5 of 13 replays. The system compared the edited skill against 13 recorded cases using deterministic side-effect checks. Five replays produced new calls to github.close_issue and github.comment that the original runs never made, and the sandbox refused them. Eight replays passed without new side effects. The edit introduces unauthorized write actions across multiple repositories. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_f8c354ae33e9", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 122, "usd": 0.00009618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_478d9ee2ce19", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 167, "usd": 0.00011238 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_755deee2dac3", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ce4b9acb3157", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 3 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 3, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_11ee7c7f4f30", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_9432a20fd19d", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1419, "completion_tokens": 153, "usd": 0.00012186 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_715dfb3b7eec", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_5ba0f0c4e92d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_39066d273f98", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 197, "usd": 0.00013218 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_c944d9078bcc", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1396, "completion_tokens": 147, "usd": 0.00011904 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_56c2b63686fb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 101, "usd": 0.00009468 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a8e17ca9255f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_dd5897c02420", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 98, "usd": 0.00009474 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00085782, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.01324, "spent_usd_by_tier": { "nano": 0.00085782, "super": 0.0, "ultra": 0.000757 }, "prompt_tokens_by_tier": { "nano": 10069, "super": 0, "ultra": 487 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 4.093, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-1", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_3a84ec6a6e84", "verdict_id": "verdict_3a84ec6a6e84", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system ran 13 replay pairs against the proposed change and found 2 regressions. One replay added issue references that were not in the expected output. Another replay attempted a new GitHub comment call that the sandbox rejected. The remaining 11 pairs passed. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_165175858316", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 128, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_187d5cdcf6dc", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay mentions issue #8: bug and #11: enhancement, which adds extra content not present in the golden output is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1228, "completion_tokens": 138, "usd": 0.0001068 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3f0a0fcb0075", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 175, "usd": 0.0001236 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_9972d203f927", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1361, "completion_tokens": 137, "usd": 0.00011454 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c71e5c2370b8", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 171, "usd": 0.00011856 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_eb9d9e6851a5", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 187, "usd": 0.00012792 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_de7759ea9e17", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_092b5ebe7778", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1350, "completion_tokens": 138, "usd": 0.00011412 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_968755dee57c", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1413, "completion_tokens": 178, "usd": 0.0001275 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_10dca42d0cf4", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1392, "completion_tokens": 126, "usd": 0.00011376 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_990c76131fc0", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_47c34378e66c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1182, "completion_tokens": 141, "usd": 0.00010476 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_602f5ee8bbac", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 98, "usd": 0.00009618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00134148, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.020643, "spent_usd_by_tier": { "nano": 0.00134148, "super": 0.0, "ultra": 0.000573 }, "prompt_tokens_by_tier": { "nano": 15498, "super": 0, "ultra": 330 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.974, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_b1b8e77cd56f", "verdict_id": "verdict_b1b8e77cd56f", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 14 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 14 replays. Review before merging. The system replayed 14 test cases against the edited fx-expense-converter skill, resulting in 12 passes and 2 regressions. The first failure occurs when converting at a weekend date; the exchange rate applied uses 2026-10-09 instead of the required 2026-10-11, shifting the value from 1.0768 to 1.0800. The second failure rewrites a sentence formally but adds extra content that violates the no-additional-content rule. No drift, uncertain, or improved pairs were detected. Cost was negligible. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_9b9fe0986841", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1324, "completion_tokens": 129, "usd": 0.0001104 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_f97cae02fbc4", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The date in the rate used is 2026-10-09 instead of the required 2026-10-11 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1352, "completion_tokens": 144, "usd": 0.00011568 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6eb01a2d41d3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1342, "completion_tokens": 161, "usd": 0.00011916 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_315104ec0683", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 102, "usd": 0.0001074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_762ca612dbe8", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 263, "usd": 0.00014454 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_4ef2ca8f8b72", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1396, "completion_tokens": 205, "usd": 0.00013296 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_53e1872e54f8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 219, "usd": 0.00013896 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_fddd61f372fe", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 219, "usd": 0.0001317 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_bda2112c7543", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 118, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c3051db65b47", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 217, "usd": 0.0001368 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_8981ccff882f", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1650, "completion_tokens": 136, "usd": 0.00013164 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_ec291cce2b97", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The output includes additional content beyond the formal statement, violating the 'no additional content' criterion is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 99, "usd": 0.000093 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_cbb67ab2491c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_91fcb24262ed", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 64, "usd": 0.0000873 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00165246, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.025393, "spent_usd_by_tier": { "nano": 0.00170274, "super": 0.0001899, "ultra": 0.000844 }, "prompt_tokens_by_tier": { "nano": 19651, "super": 588, "ultra": 358 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 10.095, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_96ec8b7dc6d1", "verdict_id": "verdict_96ec8b7dc6d1", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system replayed 13 test pairs against the edited skill and found 12 passing and 1 regression. The regression occurred on \"Compute 17% of 240\" where the output lost required bold formatting around the result 40.8. The change modified the skill markdown with two additions and two deletions. Cost was $0.0015. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_382f2e4d44ae", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1300, "completion_tokens": 135, "usd": 0.0001104 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_75e2cc6a76a8", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 125, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_c536f0cd0389", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 180, "usd": 0.00012216 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_a8c493d9890b", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 100, "usd": 0.00010692 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_b7a6a78b225c", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1345, "completion_tokens": 226, "usd": 0.00013494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a00a986b53f7", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1383, "completion_tokens": 141, "usd": 0.00011682 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_c7bcfb7bf68e", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1428, "completion_tokens": 193, "usd": 0.000132 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_3b832d8b5c25", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1367, "completion_tokens": 218, "usd": 0.00013434 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_f76c8a253ef0", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 119, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c432f688c91f", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 225, "usd": 0.000138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1f1fa89dddfb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1150, "completion_tokens": 97, "usd": 0.00009228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_9470f1a93d4f", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The golden final output includes \"= **40.8**\" (bold) while the replay output shows \"= 40.8\" (no bold). This removes the required phrase \"= **40.8**\" from the output is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 127, "usd": 0.00009852 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f86b4f70f4f6", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 101, "usd": 0.00009402 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0015072, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.023133, "spent_usd_by_tier": { "nano": 0.0015072, "super": 0.0, "ultra": 0.000636 }, "prompt_tokens_by_tier": { "nano": 17172, "super": 0, "ultra": 306 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 6.755, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-1", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_1ab88c7fef08", "verdict_id": "verdict_1ab88c7fef08", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 3 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 3 of 22 replays. Review before merging. The edit to expense-report changed one line in SKILL.md. Automated replay ran 22 test pairs: 19 passed, 3 regressed, none drifted or improved. The three regressions are in \"Clean support/tickets.csv\" where row count dropped from three to four, \"Clean payments.csv\" where input rows fell from five to four, and \"Clean a header-only CSV\" where zero rows were processed. These indicate the change breaks existing cleaning logic for normal and edge-case inputs. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_ddb84b0fe47b", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1178, "completion_tokens": 124, "usd": 0.00010044 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_7d67f2353f20", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 115, "usd": 0.00011202 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_acc26084d722", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1658, "completion_tokens": 134, "usd": 0.00013164 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_af5ade88f377", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 127, "usd": 0.00010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_a69ed051d7f4", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1441, "completion_tokens": 141, "usd": 0.0001203 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_3e7d7797755d", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1430, "completion_tokens": 163, "usd": 0.00012492 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_079c36312676", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1303, "completion_tokens": 125, "usd": 0.00010818 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_f07720304e5f", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1296, "completion_tokens": 168, "usd": 0.00011808 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_8626f6fe5abb", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 125, "usd": 0.00012126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_521e59e6091f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1435, "completion_tokens": 175, "usd": 0.0001281 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_e31167be7710", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 143, "usd": 0.00010686 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_229e9b0e57cc", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 212, "usd": 0.00012714 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_1aec4a330ef6", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 106, "usd": 0.00010014 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_5b8721009163", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 170, "usd": 0.00011514 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3aee4f6f9c1d", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay reports 4 rows in the cleaned file, but the success criteria require exactly 3 rows is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 104, "usd": 0.000102 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_838608cb3f0e", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Rows in differs: golden says 5, replay says 4, indicating a different input size is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 166, "usd": 0.00011688 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_010f6ae26a1d", "task_id": "csv-header-only", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay shows zero rows in and out, meaning no data was processed, so the resulting leads.clean.csv cannot contain any rows with a non‑empty id or meet the duplicate‑row or id‑filtering criteria is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 214, "usd": 0.00012702 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_a8ba615a5d23", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1313, "completion_tokens": 200, "usd": 0.00012678 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_22e0e0ff85d9", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 182, "usd": 0.00011862 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_15046a030360", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3ac334e2f481", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb845c98bd90", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1175, "completion_tokens": 100, "usd": 0.0000945 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00248832, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.038306, "spent_usd_by_tier": { "nano": 0.00248832, "super": 0.0, "ultra": 0.000782 }, "prompt_tokens_by_tier": { "nano": 28808, "super": 0, "ultra": 380 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 8.775, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-expense-report-1", "skill": "expense-report", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_04632e3284a9", "verdict_id": "verdict_04632e3284a9", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 7 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 7 of 22 replays. Review before merging. The system replayed 22 test pairs against the edited expense-report skill, resulting in 15 passes and 7 regressions. Every regression shows that VAT amounts or explicit \"not found\" statements are now missing from the output, and the payments cleaning step no longer reports correct row counts. These are functional degradations, not formatting changes. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_73fd927ba82e", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 107, "usd": 0.00009492 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_71599b6cf2f5", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The golden output explicitly states that the second receipt was not found, which satisfies the criterion 'states the second receipt was not found' is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 193, "usd": 0.00013026 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_68029dbf8d42", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1621, "completion_tokens": 120, "usd": 0.00012606 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_0061479721a1", "task_id": "exp-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The VAT amount is missing from the output; the golden output lists VAT: 18.81 EUR but the replay only mentions \"Total: 117.81 EUR (includes VAT)\" without explicitly stating the VAT amount is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1293, "completion_tokens": 125, "usd": 0.00010758 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_cde84ba8b5d5", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1427, "completion_tokens": 159, "usd": 0.00012378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_d2bfe3e90095", "task_id": "exp-dinner-parking", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The VAT amounts are missing from the merchant lines is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 87, "usd": 0.00010578 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_ea8d1c2b4f4b", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 82, "usd": 0.00009672 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_bcff8a71d40d", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The VAT amount is omitted from the output, failing the criterion that the VAT must be stated is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1312, "completion_tokens": 107, "usd": 0.0001044 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_49a1a41ef5ee", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The VAT amounts are missing from the merchant lines; they are described as \"(includes VAT)\" instead of the explicit VAT values required is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 1507, "completion_tokens": 121, "usd": 0.00011946 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_87ffd7a9b61f", "task_id": "exp-berlin-trip", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The VAT amounts are missing from the merchant lines, violating the requirement to state the VAT for each receipt is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1444, "completion_tokens": 110, "usd": 0.00011304 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_bfab509e3676", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1257, "completion_tokens": 146, "usd": 0.00011046 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_ae36c7d648f7", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 227, "usd": 0.00012858 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_c94efe16dd1d", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 148, "usd": 0.00011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ffd4fd935f19", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1263, "completion_tokens": 183, "usd": 0.0001197 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_699475671d46", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 157, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_bbc5ac91d665", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Golden output reports 5 rows in, 3 rows out, 1 duplicate removed, 1 row without id removed is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 173, "usd": 0.00011784 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_657e87d6ef5e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 210, "usd": 0.00012606 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_8d54d0935d9d", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 115, "usd": 0.00010494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_cbe81857d988", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 117, "usd": 0.0001023 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_db46f3b0746f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 121, "usd": 0.00009732 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_10a2feb40124", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_da322788c4ba", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 119, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00244068, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.037679, "spent_usd_by_tier": { "nano": 0.00244068, "super": 0.0, "ultra": 0.000867 }, "prompt_tokens_by_tier": { "nano": 28682, "super": 0, "ultra": 564 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 8.762, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-2", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_d80f81d12a89", "verdict_id": "verdict_d80f81d12a89", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system replayed 13 test cases against the proposed change and found 4 regressions. One case, acme/docs-site, produced a new github.comment call that the original run never made, which the sandbox rejected. Three other cases lost required output: acme/data-pipeline no longer mentions exactly six label changes, acme/billing-api drops the request for details on issue #36, and a generic triage case stops saying \"repository name.\" Nine cases still pass. The failures show the edit breaks existing behavior in multiple distinct scenarios. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5bbf428edf52", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay output says \"repo name\" instead of \"repository name\" is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 86, "usd": 0.00008868 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f2d56880f7aa", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 138, "usd": 0.00010542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_c651c351049c", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 149, "usd": 0.0001188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_b6ff0cb2781b", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac63d98c8094", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1314, "completion_tokens": 150, "usd": 0.00011484 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_537a1694bbb0", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits the required mention that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1395, "completion_tokens": 95, "usd": 0.0001065 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_9e2fa3314653", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d200a62ffe51", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 142, "usd": 0.00011484 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_465ab5c43f9c", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Missing mention of issue #36 being asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1401, "completion_tokens": 117, "usd": 0.00011214 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_19dcfee12b9c", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 271, "usd": 0.00014904 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f59a42a6f25f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 91, "usd": 0.00009084 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f5acfa748023", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 80, "usd": 0.00008868 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_de28baa02c6f", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 129, "usd": 0.00010362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0011934, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.018442, "spent_usd_by_tier": { "nano": 0.0011934, "super": 0.0, "ultra": 0.000798 }, "prompt_tokens_by_tier": { "nano": 14098, "super": 0, "ultra": 381 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 6.024, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-2", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_f355647263df", "verdict_id": "verdict_f355647263df", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 4 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 4 of 13 replays. Review before merging. The system replayed 13 golden cases for the github-triage skill after a documentation update. Nine passed, but four regressed: acme/infra-terraform and acme/billing-api added an unexpected \"Closed: none\" line; acme/data-pipeline omitted the required count of six label changes; acme/mobile-app failed to explicitly state issue #58 was labeled as a bug. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0a31fbd0f6ab", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 149, "usd": 0.0001041 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e3ade3806218", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1185, "completion_tokens": 132, "usd": 0.00010278 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e7ec930d750a", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1359, "completion_tokens": 153, "usd": 0.00011826 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_c0c9ec6a7dc8", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay adds the line \"Closed: none\" which was not present in the golden final output and was not required by the success criteria is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1353, "completion_tokens": 139, "usd": 0.00011454 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_9484d74e29ab", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1330, "completion_tokens": 189, "usd": 0.00012516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_dcfc05cb8a6e", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits the required mention that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1403, "completion_tokens": 117, "usd": 0.00011226 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_b752b4bba738", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 171, "usd": 0.00012642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_811329a74d4f", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits the explicit mention that #58 is labeled as bug; it only lists #58 under 'Already labeled' without stating its label is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1353, "completion_tokens": 126, "usd": 0.00011142 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9dd1c1c9ae58", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The final output includes an extra line 'Closed: none' which was not required by the success criteria is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 102, "usd": 0.00010734 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d9834ffa3e3a", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1463, "completion_tokens": 130, "usd": 0.00011898 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3f0ff18aec4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 102, "usd": 0.00009348 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_5b3dba8d5830", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 115, "usd": 0.0000978 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f28450be2e37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142416, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.022029, "spent_usd_by_tier": { "nano": 0.00142416, "super": 0.0, "ultra": 0.000766 }, "prompt_tokens_by_tier": { "nano": 16908, "super": 0, "ultra": 424 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.006, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_f4e87eb1b76f", "verdict_id": "verdict_f4e87eb1b76f", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system replayed 13 test pairs against the updated fx-expense-converter skill, finding 12 passing and 1 regression. The failure occurred on \"Compute 17% of 240\" where the output lost required bold formatting around the result 40.8. The change modified SKILL.md with one addition and one deletion. Cost was $0.0015. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_bf3fa0074e07", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 132, "usd": 0.0001104 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_e649669b0a50", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 160, "usd": 0.00011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6b67a590cae3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 171, "usd": 0.00012144 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_1f156a1c8f92", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1370, "completion_tokens": 111, "usd": 0.00010884 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3e98cef87a9d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 230, "usd": 0.00013662 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9ffedc5fd087", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1372, "completion_tokens": 150, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_941456f71c6b", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1439, "completion_tokens": 178, "usd": 0.00012906 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_4edc319d8099", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 210, "usd": 0.0001317 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5ece1e8c9118", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 119, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_400354316051", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1388, "completion_tokens": 132, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_6cfa1d232eb4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_13feb5997c98", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The golden final output includes \"= **40.8**\" (bold formatting) while the replay output shows \"= 40.8\" (no bold). This removes the required phrase \"= **40.8**\" from the output is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 127, "usd": 0.00010068 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e2f6c6c99b4", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 120, "usd": 0.00010074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00149736, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.023016, "spent_usd_by_tier": { "nano": 0.00149736, "super": 0.0, "ultra": 0.000652 }, "prompt_tokens_by_tier": { "nano": 17196, "super": 0, "ultra": 307 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.651, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-expense-report-1", "skill": "expense-report", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_e070d22d76e7", "verdict_id": "verdict_e070d22d76e7", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 22 replays. Review before merging. The edit to expense-report SKILL.md was tested against 22 golden replays; 21 passed but the \"Clean support/tickets.csv\" replay regressed by adding an unnecessary shell.run read operation at step t002. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_60c12e465a52", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1129, "completion_tokens": 102, "usd": 0.00009222 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_87b3a1580656", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 137, "usd": 0.00011604 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_ac01bb0f765a", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1602, "completion_tokens": 117, "usd": 0.0001242 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_ba09cf46ac3d", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 169, "usd": 0.00011718 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_ca6090d7b4f3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1453, "completion_tokens": 145, "usd": 0.00012198 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_f88f40b0fdd4", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 212, "usd": 0.0001374 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_4c04fd872a46", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1303, "completion_tokens": 179, "usd": 0.00012114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a5beee38284e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 143, "usd": 0.0001128 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37245d26f903", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1509, "completion_tokens": 175, "usd": 0.00013254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_4e2b952bb502", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1447, "completion_tokens": 123, "usd": 0.00011634 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_803282d49f8b", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 164, "usd": 0.00011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_c5f95c50e3d5", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1223, "completion_tokens": 169, "usd": 0.00011394 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_8a5a0ea0ed7f", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 150, "usd": 0.0001116 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_812811600cab", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 140, "usd": 0.00010758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_eb78878a2ed4", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay adds an extra shell.run (t002) that was not present in the golden tool calls, indicating an additional read operation that is not required for cleaning the CSV is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1269, "completion_tokens": 208, "usd": 0.00012606 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0932e89999f7", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 123, "usd": 0.00010512 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_154346b1b35e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 200, "usd": 0.00012438 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_388513c72ff0", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1361, "completion_tokens": 165, "usd": 0.00012126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_1d8ff02cb8d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 101, "usd": 0.0000999 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_43b809e3abdd", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_62160dffa87e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 141, "usd": 0.00010404 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f2e5587a73ca", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 113, "usd": 0.0000984 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0025107, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.038571, "spent_usd_by_tier": { "nano": 0.0025107, "super": 0.0, "ultra": 0.000529 }, "prompt_tokens_by_tier": { "nano": 28749, "super": 0, "ultra": 289 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 9.352, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-2", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_098e7f61a6a1", "verdict_id": "verdict_098e7f61a6a1", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The system replayed 13 test pairs against the edited csv-cleaner skill, resulting in 11 passes and 2 regressions. The first regression occurred on \"Clean exports/orders.csv\" where the cleaned output dropped to 4 rows, failing the success criterion. The second regression on \"Compute 17% of 240\" showed the expected phrase \"= **40.8**\" missing from the output. Cost was $0.0014. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_959e7e0816aa", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 167, "usd": 0.0001143 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_b81786a373a5", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 153, "usd": 0.00011178 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_1dca9722e993", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 173, "usd": 0.00011406 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_96dcc8778ae9", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 221, "usd": 0.00012936 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_130fa3316eae", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 94, "usd": 0.00009888 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_b7dba255cdef", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 199, "usd": 0.00012366 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_fe6d06e8fe47", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 108, "usd": 0.00010002 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_5d23e89670d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 103, "usd": 0.00009894 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_02fdaa1af68e", "task_id": "csv-orders", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The cleaned file now has only 4 rows, but the success criteria require exactly 4 rows in the cleaned file. However, the replay shows \"Rows in: 4\" which indicates the input file has 4 rows, not 5 as in the golden output is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1275, "completion_tokens": 249, "usd": 0.00013626 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7c626553532c", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 151, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8b65fb6e74ff", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 126, "usd": 0.00009924 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f33745e12a8e", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The phrase \"= **40.8**\" is missing from the output, violating the fourth success criterion is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 97, "usd": 0.00009276 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_690661728b24", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 82, "usd": 0.00008946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00141966, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.021738, "spent_usd_by_tier": { "nano": 0.00141966, "super": 0.0, "ultra": 0.000746 }, "prompt_tokens_by_tier": { "nano": 15969, "super": 0, "ultra": 353 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 6.416, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-2", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_e7fdd47ad4bf", "verdict_id": "verdict_e7fdd47ad4bf", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 6 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 6 of 13 replays. The system compared the edited skill against the original across 13 recorded runs. Six replays produced new github.comment calls that the original runs never made; the sandbox rejected those calls. The other seven replays passed without new side effects. The change introduces unauthorized commenting behavior. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_aeb36d32b26b", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 119, "usd": 0.0000969 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_fb3621bc1546", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 165, "usd": 0.0001119 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_a026774bb5dc", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1348, "completion_tokens": 209, "usd": 0.00013104 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_082c7d58917e", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_e82538fe8409", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_4281d13e3b01", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7ee8c01d51f9", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4f8712b7b45d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_650cb16fa7da", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_6f78304e2ae4", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_96b50023c3c8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_575a1d924d31", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 141, "usd": 0.00010404 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8419e2ca5496", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 117, "usd": 0.00010002 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00063642, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.009758, "spent_usd_by_tier": { "nano": 0.00063642, "super": 0.0, "ultra": 0.000764 }, "prompt_tokens_by_tier": { "nano": 7211, "super": 0, "ultra": 518 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 4.233, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-2", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_7105394e46c3", "verdict_id": "verdict_7105394e46c3", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The system replayed 13 test cases against the updated github-triage skill and compared results to the previous version. Eleven cases passed unchanged. Two regressions were detected: a triage case for acme/search-indexer now labels issue #502 as an enhancement instead of the required question label, and a math case fails to wrap the result in bold markdown. No drift, improvements, or uncertain results were found. The change cost $0.0014. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5da690453eaf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 132, "usd": 0.00010074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_66be25a1b831", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1206, "completion_tokens": 167, "usd": 0.00011244 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_9803f74fad28", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Issue #502 is labeled as enhancement instead of question, violating the success criterion that it must be labeled as question is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1406, "completion_tokens": 132, "usd": 0.00011604 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_287402e48fdd", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1337, "completion_tokens": 137, "usd": 0.0001131 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ad012c68b2ce", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1290, "completion_tokens": 127, "usd": 0.00010788 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_c31ee7fd6635", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1402, "completion_tokens": 138, "usd": 0.00011724 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_a3e558951509", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1430, "completion_tokens": 163, "usd": 0.00012492 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7af9bb18724a", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1345, "completion_tokens": 99, "usd": 0.00010446 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9ea5f7e643be", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 176, "usd": 0.00012564 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_acc745f38e82", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 127, "usd": 0.0001152 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3ac050839ba", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 96, "usd": 0.00009132 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1e223b9eccf6", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Missing bold markdown around the result value is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 73, "usd": 0.00008772 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a03ddffaa752", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 121, "usd": 0.0001017 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014184, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.021952, "spent_usd_by_tier": { "nano": 0.0014184, "super": 0.0, "ultra": 0.000693 }, "prompt_tokens_by_tier": { "nano": 16888, "super": 0, "ultra": 309 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.416, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-1", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_8e592b2efa42", "verdict_id": "verdict_8e592b2efa42", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system replayed 13 test cases against the edited github-triage skill and detected one regression where the output incorrectly states \"4 open issues\" instead of the required \"5 open issues\" for the acme/billing-api triage case, failing the first success criterion. The remaining 12 cases passed. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_bd63d6f067f1", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 117, "usd": 0.00009498 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f6019126d256", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1181, "completion_tokens": 164, "usd": 0.00011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_dd87261b8ba6", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 133, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_2d58c0602c67", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1368, "completion_tokens": 127, "usd": 0.00011256 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c9de1093aa99", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1307, "completion_tokens": 127, "usd": 0.0001089 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5101ae1012ad", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 185, "usd": 0.00012882 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_47ce9e153501", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1393, "completion_tokens": 176, "usd": 0.00012582 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4db894881c54", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1322, "completion_tokens": 99, "usd": 0.00010308 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_c8c4828aae44", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The output states '4 open issues' instead of the required '5 open issues', failing the first success criterion is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1409, "completion_tokens": 161, "usd": 0.00012318 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_b3c44e403f10", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 146, "usd": 0.00011904 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f3095a10d7d9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_4519fd683ef9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 68, "usd": 0.00008508 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_72d1ef164d37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 122, "usd": 0.00010122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142038, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.02195, "spent_usd_by_tier": { "nano": 0.00142038, "super": 0.0, "ultra": 0.000564 }, "prompt_tokens_by_tier": { "nano": 16781, "super": 0, "ultra": 279 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.39, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-1", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_74b936184be7", "verdict_id": "verdict_74b936184be7", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system replayed 13 test cases against the edited github-triage skill; 12 passed but the \"Triage acme/data-pipeline\" case regressed because the agent omitted two required github.label calls needed to apply all six label changes. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_36523cbf4c92", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1127, "completion_tokens": 124, "usd": 0.00009738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e70cb39d820f", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 135, "usd": 0.0001047 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_93bcdbcd3914", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 133, "usd": 0.00011322 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ba25466de412", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1385, "completion_tokens": 180, "usd": 0.0001263 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_eb462443ad9b", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1295, "completion_tokens": 177, "usd": 0.00012018 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5d5629ab1937", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay omits two required github.label calls (t005 and t006) that are needed to apply all six label changes is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1405, "completion_tokens": 131, "usd": 0.00011574 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_37a737ab09a1", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1576, "completion_tokens": 169, "usd": 0.00013512 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a8f42769e490", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1329, "completion_tokens": 142, "usd": 0.00011382 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dbbba87c46a3", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1401, "completion_tokens": 177, "usd": 0.00012654 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_84d120ce3156", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1570, "completion_tokens": 144, "usd": 0.00012876 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f1c02a8e442b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1162, "completion_tokens": 97, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_227ef3c2f411", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 73, "usd": 0.00009648 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_ed4fe7d3e21d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 103, "usd": 0.00009594 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00146718, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.022668, "spent_usd_by_tier": { "nano": 0.00146718, "super": 0.0, "ultra": 0.000535 }, "prompt_tokens_by_tier": { "nano": 17313, "super": 0, "ultra": 286 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.162, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_eca751fb89a7", "verdict_id": "verdict_eca751fb89a7", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system ran 13 evaluation pairs against the updated fx-expense-converter skill, resulting in 12 passes and 1 regression. The failure occurred on the \"Convert at a weekend date\" test where the applied exchange rate date shifted from 2026-10-11 to 2026-10-09, changing the rate value from 1.0768 to 1.0800. This indicates the recent SKILL.md change introduced a date-handling defect for weekend conversions. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e5b902ded754", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 135, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_8c4cb14a441e", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The date in the rate used line is 2026-10-09 instead of the required 2026-10-11 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 111, "usd": 0.00010704 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_3f28b051e158", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1328, "completion_tokens": 202, "usd": 0.00012816 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_bdb089b8bdbb", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 102, "usd": 0.00010524 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_2393bfd5f040", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1393, "completion_tokens": 235, "usd": 0.00013998 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a029bb9d7e18", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1371, "completion_tokens": 139, "usd": 0.00011562 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_148da7df7bf5", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1452, "completion_tokens": 135, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_c24c16f3b5fa", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 158, "usd": 0.00011778 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_b4e1ea78a932", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 121, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_cd2b69e99407", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 139, "usd": 0.00011808 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_83397ea2c263", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 108, "usd": 0.0000942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_37f7a8b8bf4c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8108d03dbe59", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 110, "usd": 0.00009696 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014559, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.022498, "spent_usd_by_tier": { "nano": 0.00151398, "super": 0.0001899, "ultra": 0.000758 }, "prompt_tokens_by_tier": { "nano": 18033, "super": 588, "ultra": 326 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.202, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-2", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_b8046b191573", "verdict_id": "verdict_b8046b191573", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 3 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 3 of 22 replays. Review before merging. The system replayed 22 test pairs against the edited expense-report skill, resulting in 19 passes and 3 regressions. The regressions show missing statements for unfound receipts, extra shell steps during inventory cleaning, and incorrect row counts during payment cleaning. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_efb6b4383a27", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1319, "completion_tokens": 90, "usd": 0.00010074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_26a650f173a0", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Missing statement that the second receipt was not found is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1599, "completion_tokens": 114, "usd": 0.0001233 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_90c84e6db9bc", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1723, "completion_tokens": 74, "usd": 0.00012114 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_9f99d91e21f6", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1471, "completion_tokens": 131, "usd": 0.0001197 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_29cce0ac5de3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1611, "completion_tokens": 100, "usd": 0.00012066 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_21bf7c25b237", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1612, "completion_tokens": 118, "usd": 0.00012504 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_256b30ee0efe", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1473, "completion_tokens": 108, "usd": 0.0001143 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_dfea8f3d0de4", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 135, "usd": 0.00011892 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_f1ddb4f3653c", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1667, "completion_tokens": 110, "usd": 0.00012642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_ef0acc9e5a05", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1629, "completion_tokens": 128, "usd": 0.00012846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_11286bfe394d", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1355, "completion_tokens": 152, "usd": 0.00011778 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_325c7d095b64", "task_id": "csv-inventory", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay adds an extra shell.read step (t002) and a second shell.run (t003) that were not present in the golden tool calls, indicating a change in the processing pipeline is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1426, "completion_tokens": 165, "usd": 0.00012516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_2a529632a34a", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 105, "usd": 0.0001101 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_908da659aa18", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1373, "completion_tokens": 91, "usd": 0.00010422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3a7988db6226", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1454, "completion_tokens": 83, "usd": 0.00010716 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_3186365c17c2", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Golden output reports 4 rows in, but success criteria require exactly 3 rows in is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1442, "completion_tokens": 88, "usd": 0.00010764 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_780ead4737e0", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 145, "usd": 0.00011922 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_0ba4c66e04ad", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1471, "completion_tokens": 106, "usd": 0.0001137 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_58c7d9abe244", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 154, "usd": 0.00012138 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8e9bcc22b033", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1308, "completion_tokens": 79, "usd": 0.00009744 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1cc2a5f894e5", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1328, "completion_tokens": 79, "usd": 0.00009864 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_07aa4d542d5a", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 76, "usd": 0.00010038 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0025215, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.039594, "spent_usd_by_tier": { "nano": 0.0025215, "super": 0.0, "ultra": 0.000625 }, "prompt_tokens_by_tier": { "nano": 32301, "super": 0, "ultra": 367 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.568, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_f320edf053a3", "verdict_id": "verdict_f320edf053a3", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system evaluated the fx-expense-converter skill against 13 test pairs, resulting in 12 passes and 1 regression. The failure occurred on the \"Summarize a short note\" test where the word \"stays\" was replaced with \"will stay,\" shifting the meaning from a definite statement to a future intention and violating the criterion that the agenda remains unchanged. This single regression triggered the hold decision despite the majority of tests passing. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_8c2c5e26d4e1", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1506, "completion_tokens": 110, "usd": 0.00011676 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_2a31e3e43325", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1511, "completion_tokens": 91, "usd": 0.0001125 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_40498db1a5f2", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1486, "completion_tokens": 140, "usd": 0.00012276 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_0f0b98c0dda1", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1706, "completion_tokens": 100, "usd": 0.00012636 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_a341fb0948fa", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1539, "completion_tokens": 80, "usd": 0.00011154 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_b5a453299350", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1553, "completion_tokens": 102, "usd": 0.00011766 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ea5e1042bbf8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1597, "completion_tokens": 88, "usd": 0.00011694 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ab59cb62584a", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1489, "completion_tokens": 77, "usd": 0.00010782 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_d101d55224e4", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1586, "completion_tokens": 112, "usd": 0.00012204 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_f96f5f3cef70", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1558, "completion_tokens": 151, "usd": 0.00012972 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_eb8645af7d9f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1344, "completion_tokens": 80, "usd": 0.00009984 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_710563b99aef", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 92, "usd": 0.00010104 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb5c11b812f5", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The word \"stays\" is replaced with \"will stay\", which changes the meaning from a definite statement to a future intention, violating the criterion that the agenda is unchanged is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1358, "completion_tokens": 156, "usd": 0.00011892 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0015039, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.023686, "spent_usd_by_tier": { "nano": 0.0015039, "super": 0.0, "ultra": 0.000653 }, "prompt_tokens_by_tier": { "nano": 19549, "super": 0, "ultra": 287 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.956, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_8b357d526eda", "verdict_id": "verdict_8b357d526eda", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 5 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 5 of 13 replays. Review before merging. The system re-ran 13 test cases for the fx-expense-converter skill after a documentation change. Eight passed, three regressed, and two were uncertain due to model unavailability. The regressions show missing USD output for a Tokyo dinner, an extra finance rate call for a Zurich taxi, and missing exchange rate data for a Berlin trip. Two cases could not be judged because the model was unavailable. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e9f830fa642e", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1516, "completion_tokens": 92, "usd": 0.00011304 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_bbae78eada7e", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1520, "completion_tokens": 79, "usd": 0.00011016 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_518252b4621f", "task_id": "fx-tokyo", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The USD amount is missing from the replay output is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1496, "completion_tokens": 111, "usd": 0.0001164 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "0.0067", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_6bd370a9ae4d", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1562, "completion_tokens": 77, "usd": 0.0001122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_4a19b9d5037d", "task_id": "fx-zurich", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay tool calls include an extra t002 finance.rate (read) that is not present in the golden tool calls, indicating a change in the process is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1561, "completion_tokens": 167, "usd": 0.00013374 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.15714", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_659d9e0a0119", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1576, "completion_tokens": 134, "usd": 0.00012672 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ae6602b84c3d", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1605, "completion_tokens": 125, "usd": 0.0001263 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_76bd7f61f20c", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1547, "completion_tokens": 174, "usd": 0.00013458 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_7b5ee5b9fe63", "task_id": "fx-hamburg-conference", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_40cfb64bd6bb", "task_id": "fx-berlin-trip", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Missing exchange rate and date is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1590, "completion_tokens": 96, "usd": 0.00011844 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2c596c557862", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1308, "completion_tokens": 78, "usd": 0.0000972 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_dd37987a313e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 79, "usd": 0.00009792 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_14951b257b04", "task_id": "canary-summary", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0012867, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.020233, "spent_usd_by_tier": { "nano": 0.00152142, "super": 0.0009288, "ultra": 0.000755 }, "prompt_tokens_by_tier": { "nano": 19977, "super": 2844, "ultra": 410 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.05, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_be2076adc43a", "verdict_id": "verdict_be2076adc43a", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The tavily judge evaluated the fx-expense-converter skill against 13 replay pairs. Eleven passed, one regressed, and one was uncertain. The regression occurred on a weekend date conversion where the rate date shifted from 2026-10-11 to 2026-10-09 and the value moved from 1.0768 to 1.0800. The uncertain case involved a Zurich taxi conversion where the model was unavailable. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_88832e95c9b9", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1482, "completion_tokens": 92, "usd": 0.000111 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_33a9c6adf157", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The date in the rate used is 2026-10-09 instead of the required 2026-10-11 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1486, "completion_tokens": 125, "usd": 0.00011916 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_07551b4c2872", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1462, "completion_tokens": 129, "usd": 0.00011868 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f725843c1f80", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1564, "completion_tokens": 90, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_0545f282df0a", "task_id": "fx-zurich", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.15714", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9df69746323e", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1554, "completion_tokens": 88, "usd": 0.00011436 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_51bbbd84f8be", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1598, "completion_tokens": 82, "usd": 0.00011556 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_32ea0f012a38", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1513, "completion_tokens": 84, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_1847ee62554a", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1610, "completion_tokens": 80, "usd": 0.0001158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_08c9424f7da6", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1582, "completion_tokens": 156, "usd": 0.00013236 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_a144e4814df4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1308, "completion_tokens": 83, "usd": 0.0000984 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e625a12fe4a6", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 73, "usd": 0.00009792 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c818c2acd85b", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1358, "completion_tokens": 127, "usd": 0.00011196 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00136158, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.021484, "spent_usd_by_tier": { "nano": 0.0014781, "super": 0.00042, "ultra": 0.000752 }, "prompt_tokens_by_tier": { "nano": 19531, "super": 1292, "ultra": 350 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.27, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-2", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_043b653a4fc8", "verdict_id": "verdict_043b653a4fc8", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The regression occurred in the \"Triage acme/infra-terraform\" test where the deterministic side-effect check detected two new calls to github.comment that the original run never made, causing the sandbox to refuse them at step t006. The other 12 test pairs passed without drift or uncertainty. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_9aae4e6186d2", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1321, "completion_tokens": 92, "usd": 0.00010134 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_852cf7299d83", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1351, "completion_tokens": 107, "usd": 0.00010674 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_1f4cd7269ecb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1554, "completion_tokens": 131, "usd": 0.00012468 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d6db457362fe", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_7d366fbfea40", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1489, "completion_tokens": 111, "usd": 0.00011598 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_bb40249ba200", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1566, "completion_tokens": 108, "usd": 0.00011988 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_5cb5a86b5b0c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1551, "completion_tokens": 144, "usd": 0.00012762 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_8e097e2bc153", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1511, "completion_tokens": 146, "usd": 0.0001257 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dc72a3a720da", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1536, "completion_tokens": 150, "usd": 0.00012816 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d4e29e63139d", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1562, "completion_tokens": 122, "usd": 0.000123 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1bcf329ed33b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1332, "completion_tokens": 78, "usd": 0.00009864 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_20de0884c5fb", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1328, "completion_tokens": 73, "usd": 0.0000972 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b123eb9839bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 120, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00137988, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.021616, "spent_usd_by_tier": { "nano": 0.00137988, "super": 0.0, "ultra": 0.000546 }, "prompt_tokens_by_tier": { "nano": 17470, "super": 0, "ultra": 279 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.227, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-2", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_ebd977ff9ea8", "verdict_id": "verdict_ebd977ff9ea8", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. Ratchet replayed 13 past tasks against the edit to github-triage and compared each with its blessed result. On \"Triage without a repository\": Got worse. The golden final output explicitly requires the repository name to be in the format `acme/` and to mention that the repository is to be triaged is no longer met. On \"Triage acme/widgets\": Got worse. Missing mention of #9 being labeled is no longer met. (step t007) Judging cost $0.0015 (nano). Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0e10ad5bac34", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The golden final output explicitly requires the repository name to be in the format `acme/` and to mention that the repository is to be triaged is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1332, "completion_tokens": 177, "usd": 0.0001224 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e27ff37067f7", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1387, "completion_tokens": 140, "usd": 0.00011682 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3d9dd364ab27", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1530, "completion_tokens": 128, "usd": 0.00012252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_570420a9b31d", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1543, "completion_tokens": 133, "usd": 0.0001245 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_40e6b8dc80bd", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1474, "completion_tokens": 116, "usd": 0.00011628 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5a38101f8f14", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1586, "completion_tokens": 101, "usd": 0.0001194 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_1f166314792c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1593, "completion_tokens": 169, "usd": 0.00013614 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d9f90397db42", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1475, "completion_tokens": 121, "usd": 0.00011754 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_13d086a4d0a2", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1555, "completion_tokens": 145, "usd": 0.0001281 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_24cc0a76ffb7", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Missing mention of #9 being labeled is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1550, "completion_tokens": 85, "usd": 0.0001134 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_86211c1cba50", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1332, "completion_tokens": 97, "usd": 0.0001032 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_70087be29148", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 73, "usd": 0.00009648 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3b107f1aecf5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 71, "usd": 0.00009846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00151524, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.023698, "spent_usd_by_tier": { "nano": 0.00151524, "super": 0.0, "ultra": 0.00067 }, "prompt_tokens_by_tier": { "nano": 19030, "super": 0, "ultra": 310 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.313, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-1", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_ee14e0baa477", "verdict_id": "verdict_ee14e0baa477", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 9 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 9 of 13 replays. Review before merging. The check ran 13 replays against the edited github-triage skill. Four passed, two regressed, and seven were uncertain because the model was unavailable. The two regressions show the agent adding extra issue lines not in the required format and dropping bold markup on a math result. The seven uncertain cases cover most of the repository triage scenarios and remain unevaluated. Do not merge until the regressions are fixed and the uncertain cases are re-run with an available model. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_fc1895f58dcf", "task_id": "tri-no-repo", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_99daafbd39ef", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "The replay adds extra content \"#8: bug\" and \"#11: enhancement\" that is not part of the required output format is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1410, "completion_tokens": 133, "usd": 0.00011652 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_204e00dc72de", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d8383ef71ebb", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1538, "completion_tokens": 115, "usd": 0.00011988 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_f644709d32f1", "task_id": "tri-auth-service", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_92e4b2c50b1c", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_574e79e6ac2e", "task_id": "tri-docs-site", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7213bfe2b9f8", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_a05f5333a824", "task_id": "tri-billing-api", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_1891e80878db", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1570, "completion_tokens": 154, "usd": 0.00013116 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d5662e056952", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1308, "completion_tokens": 83, "usd": 0.0000984 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e3d06ab590fa", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Missing bold markup around the result is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1328, "completion_tokens": 79, "usd": 0.00009864 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_18d4cf624d2d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1333, "completion_tokens": 71, "usd": 0.00009702 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00066162, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.010392, "spent_usd_by_tier": { "nano": 0.00066162, "super": 0.0, "ultra": 0.000867 }, "prompt_tokens_by_tier": { "nano": 8487, "super": 0, "ultra": 483 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 12.919, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_2004fa60e74a", "verdict_id": "verdict_2004fa60e74a", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 3 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 3 of 13 replays. Review before merging. The edit changed fx-expense-converter SKILL.md. Ten replays passed, one regressed, and two were uncertain due to model unavailability. The regression shows a weekend date conversion used a rate from 2026-10-09 instead of the required 2026-10-11, with the value shifting from 1.0768 to 1.0800. The two uncertain cases involve coworking and courier costs and Hamburg conference costs. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_4e739b0fed3d", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1494, "completion_tokens": 103, "usd": 0.00011436 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_d7ede738861c", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The date in the rate used is 2026-10-09 instead of the required 2026-10-11 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1510, "completion_tokens": 118, "usd": 0.00011892 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_7f573055a9ef", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1486, "completion_tokens": 131, "usd": 0.0001206 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f4c1c98c1a21", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1552, "completion_tokens": 97, "usd": 0.0001164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_6b7462d13be4", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1551, "completion_tokens": 75, "usd": 0.00011106 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_914ca7824ded", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1554, "completion_tokens": 85, "usd": 0.00011364 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_da9dc2ce34c7", "task_id": "fx-coworking-courier", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_70ade45e8144", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1513, "completion_tokens": 84, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_0b662be246d7", "task_id": "fx-hamburg-conference", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_ca8b25ae83a2", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1570, "completion_tokens": 151, "usd": 0.00013044 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_0bda149accdc", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1320, "completion_tokens": 80, "usd": 0.0000984 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e0d76dcf56dd", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1328, "completion_tokens": 73, "usd": 0.0000972 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f3b045e8f40e", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 71, "usd": 0.00009846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00123042, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.019439, "spent_usd_by_tier": { "nano": 0.00140562, "super": 0.0006537, "ultra": 0.000782 }, "prompt_tokens_by_tier": { "nano": 18759, "super": 2008, "ultra": 371 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.891, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-expense-report-2", "skill": "expense-report", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_3d47209d916f", "verdict_id": "verdict_3d47209d916f", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 9 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 9 of 22 replays. Review before merging. The system replayed 22 test pairs for the expense-report skill after a one-line change to SKILL.md. Thirteen passed, three regressed, and six were marked uncertain because the judge model was unavailable. The clearest regressions show missing receipt filing, extra shell commands during CSV cleaning, and an incorrect row count of four instead of three. The uncertain cases cover CSV cleaning, text rewriting, and a math calculation, all held for human review. Cost was negligible. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2f14f21d6d9f", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1276, "completion_tokens": 145, "usd": 0.00011136 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_e92a7282295d", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Missing filing of second receipt is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1586, "completion_tokens": 99, "usd": 0.00011892 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_a6b930ea6aff", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1824, "completion_tokens": 128, "usd": 0.00014016 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_e2914d406490", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 100, "usd": 0.00011154 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_07a85852522a", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1599, "completion_tokens": 100, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_85a67c3ff72c", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1624, "completion_tokens": 116, "usd": 0.00012528 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_afcfe0768c5a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1461, "completion_tokens": 144, "usd": 0.00012222 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a64afbf6b2cb", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 111, "usd": 0.00011316 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_9c6ec62f2eb9", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1691, "completion_tokens": 112, "usd": 0.00012834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6ad0a889dbc3", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1641, "completion_tokens": 126, "usd": 0.0001287 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_b6e697e319df", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1403, "completion_tokens": 94, "usd": 0.00010674 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_8ffffbf26485", "task_id": "csv-inventory", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The replay adds an extra shell.read call (t002) and an extra shell.run call (t002) that were not present in the golden trace, indicating a different processing step is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1426, "completion_tokens": 187, "usd": 0.00013044 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_56a48c4930ba", "task_id": "csv-customers", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_4c8e63150ce8", "task_id": "csv-already-clean", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_aa8e42f0327e", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Rows out reported as 4 instead of required 3 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1543, "completion_tokens": 120, "usd": 0.00012138 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_c3600f8c1037", "task_id": "csv-payments", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_64e31b08b933", "task_id": "csv-header-only", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_c2e995113d65", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1459, "completion_tokens": 102, "usd": 0.00011202 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_18bb09c7d462", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1395, "completion_tokens": 116, "usd": 0.00011154 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_de7f8d3695ad", "task_id": "canary-formal-rewrite", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_40e8a755ae4f", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b58e85552abd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1369, "completion_tokens": 120, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00191268, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.029958, "spent_usd_by_tier": { "nano": 0.00191268, "super": 0.0, "ultra": 0.00089 }, "prompt_tokens_by_tier": { "nano": 24198, "super": 0, "ultra": 503 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.857, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-expense-report-1", "skill": "expense-report", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_e9e3f7ed2784", "verdict_id": "verdict_e9e3f7ed2784", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 9 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 9 of 22 replays. Review before merging. The system ran 22 test replays against the edited expense-report skill. Thirteen passed, five regressed, and four were marked uncertain because the judge model was unavailable. The five regressions all involve missing VAT amounts in receipt filing scenarios. The four uncertain cases cover CSV cleaning tasks and a taxi-hotel receipt filing. The evaluation cost $0.0021. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2cfa611323e6", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1264, "completion_tokens": 135, "usd": 0.00010824 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_f3f61876a85b", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1612, "completion_tokens": 128, "usd": 0.00012744 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_03e985bf60c4", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1759, "completion_tokens": 99, "usd": 0.0001293 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_b14c6d6a1857", "task_id": "exp-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Missing VAT amount in replay output is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1508, "completion_tokens": 85, "usd": 0.00011088 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_758cddadc695", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1683, "completion_tokens": 101, "usd": 0.00012522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5e47aca6907b", "task_id": "exp-dinner-parking", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Missing VAT amounts for each receipt is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1626, "completion_tokens": 102, "usd": 0.00012204 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a8a18a97157f", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Missing VAT amount '7.58 EUR' is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1497, "completion_tokens": 95, "usd": 0.00011262 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_2a8fe0325d1e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1490, "completion_tokens": 119, "usd": 0.00011796 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_21460568387a", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Missing explicit VAT amounts for each receipt is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1687, "completion_tokens": 252, "usd": 0.0001617 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_1b2e69a62019", "task_id": "exp-berlin-trip", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_df0530c17b4e", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 149, "usd": 0.00012066 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_5b34a19cd981", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1405, "completion_tokens": 82, "usd": 0.00010398 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_b1ea18acb350", "task_id": "csv-customers", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_6d6c20462d5a", "task_id": "csv-already-clean", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_1d5047ec8fd2", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_a7ca6fd2483c", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "regression", "decided_by": "nano", "reason": "Rows in differs (4 vs 5) is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1430, "completion_tokens": 109, "usd": 0.00011196 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_8ed25d9902cb", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1383, "completion_tokens": 149, "usd": 0.00011874 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_bdb9bd3422e8", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1471, "completion_tokens": 92, "usd": 0.00011034 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_03c42fb9fb74", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1419, "completion_tokens": 119, "usd": 0.0001137 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d54e294c1ad8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1332, "completion_tokens": 94, "usd": 0.00010248 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3cb515f6830e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1340, "completion_tokens": 116, "usd": 0.00010824 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c51d156486d2", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 71, "usd": 0.00009846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00210396, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.032969, "spent_usd_by_tier": { "nano": 0.00210396, "super": 0.0, "ultra": 0.000798 }, "prompt_tokens_by_tier": { "nano": 26678, "super": 0, "ultra": 474 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.226, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-1", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_8b34577aa3ff", "verdict_id": "verdict_8b34577aa3ff", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 5 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 5 of 13 replays. Review before merging. The system re-ran 13 test cases for the csv-cleaner skill after a documentation update. Eight cases passed, four regressed, and one was uncertain due to a model outage. The regressions show the cleaner now drops required rows or miscounts input across headers-only, support tickets, payments, and signups files. The uncertain case involved a simple math prompt the judge could not evaluate. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_95b0aeaae315", "task_id": "csv-header-only", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "No rows are written to leads.clean.csv, violating the requirement that it must contain at least one row with a non‑empty id is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1443, "completion_tokens": 172, "usd": 0.00012786 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ad0ef0f6f262", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1397, "completion_tokens": 119, "usd": 0.00011238 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_4bc22f69c2f4", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1427, "completion_tokens": 147, "usd": 0.0001209 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_341f1aafe06d", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Rows out increased from 3 to 4, violating the requirement of exactly 3 rows is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 122, "usd": 0.00011364 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_89cbeb5f1106", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Rows in differs: golden says 5 rows in, replay says 4 rows in is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1430, "completion_tokens": 112, "usd": 0.00011268 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_638b93eff172", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1459, "completion_tokens": 96, "usd": 0.00011058 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_57f1a1c928ec", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1417, "completion_tokens": 83, "usd": 0.00010494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_ed700d3411da", "task_id": "csv-signups", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Rows in: 3 differs from required 4 rows in signups.csv is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1443, "completion_tokens": 140, "usd": 0.00012018 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_f83b9f46b05b", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1441, "completion_tokens": 108, "usd": 0.00011238 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7323ba2c2619", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 108, "usd": 0.00011082 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2e08956d5846", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1344, "completion_tokens": 84, "usd": 0.0001008 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_8ad7f2f013aa", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3f0ad4c092bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1333, "completion_tokens": 71, "usd": 0.00009702 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00134418, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.021041, "spent_usd_by_tier": { "nano": 0.00134418, "super": 0.0, "ultra": 0.000774 }, "prompt_tokens_by_tier": { "nano": 16955, "super": 0, "ultra": 429 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.689, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-1", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_c8bca0d60279", "verdict_id": "verdict_c8bca0d60279", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 5 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 5 of 22 replays. Review before merging. The evaluation ran 22 test pairs against the edited expense-report skill, resulting in 17 passes, one regression, and four uncertain outcomes where the judge model was unavailable. The regression occurred in \"Clean payments.csv\" where the replay produced four rows instead of the expected five at step t001. The four uncertain cases covering receipt filing, formal rewriting, percentage calculation, and note summarization were held for human review. Cost was $0.0021. Inspect the failing replay to confirm the row count difference is a true defect. Approve only if the regression is expected and the uncertain cases are acceptable. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_9797515450fd", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1240, "completion_tokens": 133, "usd": 0.00010632 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_562f1bacbb4a", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1575, "completion_tokens": 112, "usd": 0.00012138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_118b690af2c2", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1840, "completion_tokens": 95, "usd": 0.0001332 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_17c39cd8a28a", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1483, "completion_tokens": 112, "usd": 0.00011586 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_89a8beb01ad9", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1599, "completion_tokens": 104, "usd": 0.0001209 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5a08a4d190cc", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1636, "completion_tokens": 115, "usd": 0.00012576 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a04984da2f2e", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1485, "completion_tokens": 123, "usd": 0.00011862 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_6c284a0936b6", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1502, "completion_tokens": 122, "usd": 0.0001194 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37dcfb667a2b", "task_id": "exp-lunch-train", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_f63fa53e365f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1641, "completion_tokens": 127, "usd": 0.00012894 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_596660827ddb", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 155, "usd": 0.0001221 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_35dd5c46b4f3", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 131, "usd": 0.00011718 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_24bcb89d6737", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1403, "completion_tokens": 147, "usd": 0.00011946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_3053a2b19841", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1409, "completion_tokens": 136, "usd": 0.00011718 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_df65d3ad0120", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1442, "completion_tokens": 83, "usd": 0.00010644 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_175a9b556a3d", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Rows in differs: golden says 5, replay says 4 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 100, "usd": 0.00010836 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_6396f7029b97", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1528, "completion_tokens": 181, "usd": 0.00013512 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_7ff7643575cd", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1459, "completion_tokens": 113, "usd": 0.00011466 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_07fbc9fb046d", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1419, "completion_tokens": 118, "usd": 0.00011346 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_11e6e85db7b9", "task_id": "canary-formal-rewrite", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_6781c847f496", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_16136c6d8859", "task_id": "canary-summary", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "nano", "reason": "Model unavailable; judged as uncertain. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "usd": 0.0 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00214434, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.033532, "spent_usd_by_tier": { "nano": 0.00214434, "super": 0.0, "ultra": 0.000843 }, "prompt_tokens_by_tier": { "nano": 26911, "super": 0, "ultra": 363 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.426, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-github-triage-1", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_08f35af554b6", "verdict_id": "verdict_08f35af554b6", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The system replayed 13 test cases against the updated github-triage skill and compared results to the previous version. Eleven cases passed unchanged. Two cases regressed: \"Triage acme/mobile-app\" now incorrectly labels issue #62 as \"bug\" instead of \"already labeled,\" and \"Triage acme/widgets\" fails to mention that issue #9 is already labeled. Both failures occurred at step t002. No drift, uncertain, or improved results were detected. The change cost $0.0015. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_c6a8e1abb734", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1333, "completion_tokens": 92, "usd": 0.00010206 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_4e23436145b0", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1375, "completion_tokens": 97, "usd": 0.00010578 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e1dd839734fb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1554, "completion_tokens": 110, "usd": 0.00011964 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_860252f0aeb1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1550, "completion_tokens": 122, "usd": 0.00012228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac0076e8ac87", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1477, "completion_tokens": 114, "usd": 0.00011598 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_0247b5972b80", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1578, "completion_tokens": 185, "usd": 0.00013908 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_07bb9b326000", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1648, "completion_tokens": 140, "usd": 0.00013248 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_ef274337a616", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "The label for #62 is incorrectly set to \"bug\" instead of \"already labeled\" is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1510, "completion_tokens": 120, "usd": 0.0001194 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_ee6719208401", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1579, "completion_tokens": 140, "usd": 0.00012834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_44c02d5b3570", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "nano", "reason": "Missing mention of #9 as labeled is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1559, "completion_tokens": 110, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_bfdc8aa6fc39", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1308, "completion_tokens": 96, "usd": 0.00010152 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a63316f74650", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1304, "completion_tokens": 85, "usd": 0.00009864 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_86b52a6672e1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1345, "completion_tokens": 71, "usd": 0.00009774 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00150288, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.023566, "spent_usd_by_tier": { "nano": 0.00150288, "super": 0.0, "ultra": 0.000732 }, "prompt_tokens_by_tier": { "nano": 19120, "super": 0, "ultra": 306 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.821, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_666ce8f28fc2", "verdict_id": "verdict_666ce8f28fc2", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 1 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 1 changed because the world did, not the skill. The system ran 13 evaluation pairs for the fx-expense-converter skill, resulting in 12 passes and 1 drift with no regressions. The single drift occurred in the \"Convert London costs from GBP\" task where the GBP/USD exchange rate shifted from 1.2702 to 1.3105 between runs, causing the output to update correctly. The skill logic remains sound and the change reflects an external data update. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_3e8797c16149", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 110, "usd": 0.00010656 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_608d63d15c36", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 160, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_229dc213dbd6", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 187, "usd": 0.00012312 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_e71800e8113a", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1406, "completion_tokens": 89, "usd": 0.00010572 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_16ce4eaa1d1a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 230, "usd": 0.00013734 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_bd178338e4b1", "task_id": "fx-london", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The GBP/USD exchange rate on 2026-10-13 was 1.2702 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 220, "usd": 0.00013722 } ], "control_outcome": null, "drift": [ { "fact_id": "t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.2702", "new_value": "1.3105" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_f6108436cd3f", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1451, "completion_tokens": 135, "usd": 0.00011946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_133f03cc5707", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 132, "usd": 0.00011298 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_cb6d6a6fa18e", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 124, "usd": 0.00011616 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_9bc309eb22db", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1377, "completion_tokens": 164, "usd": 0.00012198 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_c0d90d4cb42f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 97, "usd": 0.00009156 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_7d778bb3b477", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9620dc2201ff", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 122, "usd": 0.00010122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014796, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.022818, "spent_usd_by_tier": { "nano": 0.00152988, "super": 0.0001965, "ultra": 0.000702 }, "prompt_tokens_by_tier": { "nano": 17994, "super": 604, "ultra": 327 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.98, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_c072b05b1f7a", "verdict_id": "verdict_c072b05b1f7a", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 5 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 5 changed because the world did, not the skill. The evaluation ran 13 test cases against the updated skill documentation. Eight cases passed with identical results. Five cases showed drift because the EUR/USD exchange rate for 2026-10-14 moved from 1.0800 to 1.163, and the skill correctly applied the new rate. No regressions or improvements were detected. The skill logic remains sound; only the external financial data changed. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_24416e3f8916", "task_id": "fx-no-date", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1347, "completion_tokens": 132, "usd": 0.0001125 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_c04a94e117fd", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "drift", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 160, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_823b4ddae43a", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 160, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_9326a185dca9", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1402, "completion_tokens": 152, "usd": 0.0001206 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_48d13ccbd36d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 190, "usd": 0.00012774 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_24d29ab3083c", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1407, "completion_tokens": 222, "usd": 0.0001377 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_d540794f1c98", "task_id": "fx-coworking-courier", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1426, "completion_tokens": 129, "usd": 0.00011652 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ecc9e77802a1", "task_id": "fx-software", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "finance.rate returned 1.08 on 2026-10-14 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1366, "completion_tokens": 168, "usd": 0.00012228 } ], "control_outcome": null, "drift": [ { "fact_id": "obs_t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.08", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_ce1c4a74b4df", "task_id": "fx-hamburg-conference", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1439, "completion_tokens": 184, "usd": 0.0001305 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_84352baa167a", "task_id": "fx-berlin-trip", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 161, "usd": 0.00012402 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_68a265f4d0b3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "drift", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 99, "usd": 0.0000942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_eac0515b4a0c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 74, "usd": 0.00008724 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_aa0845d7fbd7", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 97, "usd": 0.00009522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00150468, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.02315, "spent_usd_by_tier": { "nano": 0.0020301, "super": 0.0009474, "ultra": 0.000989 }, "prompt_tokens_by_tier": { "nano": 23059, "super": 2903, "ultra": 629 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 7.838, "simulated_models": false, "repeat": 0, "error": null } ] }, "super_only": { "config": { "name": "super_only", "description": "Super only — every pair judged by super alone (Level 0 and drift as in the cascade).", "experiments": [ "E4" ], "mode": "edits", "cascade": { "mode": "super_only", "level0": true, "drift": true, "controls": true }, "single_judge_blocks": true, "types": [ "R1", "R2", "R3", "R4", "R5", "R6", "N0", "D1" ], "noise_repeats": 3, "runner": "local", "concurrency": 2, "reuse_replays": "cascade", "max_traces": 40 }, "config_path": "bench/configs/super_only.yaml", "command": "ratchet bench run --config bench/configs/super_only.yaml --out bench/results/2026-10-16 --types D1 --limit 2 --yes", "started_at": "2026-10-16T12:51:24.187382Z", "updated_at": "2026-10-16T13:57:13.524050Z", "edits": [ { "bench_id": "n0-github-triage-2", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_a61b3b3a525d", "verdict_id": "verdict_a61b3b3a525d", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system replayed 13 evaluation pairs against the edited github-triage skill, checking for behavioral changes in label operations and triage logic. Twelve pairs passed, confirming existing behavior is largely preserved. One regression was detected on the \"Triage acme/data-pipeline\" case where the agent failed to apply exactly six label changes at step t002, indicating a specific logic break. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_811b35141f2c", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1549, "completion_tokens": 179, "usd": 0.0006258 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_b3d50ce1b799", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2596, "completion_tokens": 213, "usd": 0.0009705 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_7741c40ef280", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3268, "completion_tokens": 302, "usd": 0.0012522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_abe3d0283a76", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3284, "completion_tokens": 351, "usd": 0.0013011 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_a0217d5141b7", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3020, "completion_tokens": 279, "usd": 0.0011571 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_f5d663dbc019", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "specifies that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3313, "completion_tokens": 347, "usd": 0.0013062 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7e7f0890e15f", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2962, "completion_tokens": 274, "usd": 0.0011352 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a1def1b764dc", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3074, "completion_tokens": 320, "usd": 0.0012102 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_0490537a6338", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3392, "completion_tokens": 288, "usd": 0.0012768 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_8b534e552950", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3197, "completion_tokens": 341, "usd": 0.001266 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_7e792837459f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1544, "completion_tokens": 193, "usd": 0.0006369 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_95212662e369", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1583, "completion_tokens": 227, "usd": 0.0006792 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a24da7604ca9", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1586, "completion_tokens": 220, "usd": 0.0006738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.013491, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.04497, "spent_usd_by_tier": { "nano": 0.0, "super": 0.013491, "ultra": 0.000593 }, "prompt_tokens_by_tier": { "nano": 0, "super": 34368, "ultra": 263 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.346, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-2", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_b80f5bf37996", "verdict_id": "verdict_b80f5bf37996", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The single-line change to SKILL.md was evaluated against the full regression suite of 22 task pairs. Every pair produced the expected output with zero regressions, zero drift, and zero uncertain results. The edit cost $0.02 in super compute. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_db2fa52f33ec", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1591, "completion_tokens": 159, "usd": 0.0006204 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_663c04fc7679", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3403, "completion_tokens": 262, "usd": 0.0012567 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_b8eacf7ca9eb", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3973, "completion_tokens": 142, "usd": 0.0013197 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_df4a9a6d2d5e", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2597, "completion_tokens": 279, "usd": 0.0010302 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_2aba1c6b9a48", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3297, "completion_tokens": 288, "usd": 0.0012483 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_4d8a32096183", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3314, "completion_tokens": 313, "usd": 0.0012759 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_5d79f77bf29a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2599, "completion_tokens": 301, "usd": 0.0010506 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_fa5cfd551a14", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2629, "completion_tokens": 281, "usd": 0.0010416 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_bcd2f6c4de1b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3485, "completion_tokens": 379, "usd": 0.0013866 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6c9bfaed6179", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3295, "completion_tokens": 314, "usd": 0.0012711 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_979bc36816b3", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2491, "completion_tokens": 252, "usd": 0.0009741 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_0444b760bf07", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2729, "completion_tokens": 293, "usd": 0.0010824 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_997e188d6e47", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2657, "completion_tokens": 232, "usd": 0.0010059 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_d40c1503ab73", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2672, "completion_tokens": 357, "usd": 0.0011229 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_8111d388d469", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2835, "completion_tokens": 275, "usd": 0.001098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0727abbc4d2a", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2770, "completion_tokens": 311, "usd": 0.0011109 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_cfff2f4aa409", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2350, "completion_tokens": 242, "usd": 0.0009228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_52606712bd67", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2658, "completion_tokens": 279, "usd": 0.0010485 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_542487f91fa5", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2708, "completion_tokens": 326, "usd": 0.0011058 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_599436fdbbc6", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1621, "completion_tokens": 202, "usd": 0.0006681 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_de6195626ea9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1720, "completion_tokens": 224, "usd": 0.0007176 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e212cacce84", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1709, "completion_tokens": 211, "usd": 0.0007026 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0230607, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.076869, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0230607, "ultra": 0.000436 }, "prompt_tokens_by_tier": { "nano": 0, "super": 59103, "ultra": 220 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.882, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-1", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_756a66380c2c", "verdict_id": "verdict_756a66380c2c", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 8 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 8 of 13 replays. Review before merging. The system re-ran 13 triage replays against the edited skill and compared each step output to the previous baseline. Five replays passed unchanged. Eight regressed, all failing on step m004 or t003 where the agent previously asked for details or labeled issues correctly but now omits those actions. The strongest evidence shows the agent no longer asks for details on issues for acme/auth-service, acme/billing-api, acme/widgets, acme/mobile-app, and acme/infra-terraform, and fails to label acme/search-indexer or format output for acme/status-page. Cost was $0.01. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_29198b3e615a", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1533, "completion_tokens": 187, "usd": 0.0006282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_ae5049772f22", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "contains no additional text that is not part of the required output format is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2370, "completion_tokens": 256, "usd": 0.0009414 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_6a4a7b2c3a20", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "labels issue #502 as question is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3159, "completion_tokens": 307, "usd": 0.001224 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ff7547adf9e1", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "Missing 'Asked for details: #77' line in the replay output is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3229, "completion_tokens": 270, "usd": 0.0012117 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_95dc2c9a2266", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "asks for details for #91 is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2941, "completion_tokens": 250, "usd": 0.0011073 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_1ac70e25ffb3", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3136, "completion_tokens": 312, "usd": 0.0012216 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_3be8109cfb2f", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states that no details were asked for is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3157, "completion_tokens": 264, "usd": 0.0011847 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_650e3f7e3ede", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "asks for details on #64 and cites github.comment as artifact changed is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3040, "completion_tokens": 262, "usd": 0.0011478 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_e5bee22bc04e", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "mentions that issue #36 was asked for details is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3433, "completion_tokens": 294, "usd": 0.0012945 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_0c922bfa825f", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "mentions #15 was asked for details is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3152, "completion_tokens": 297, "usd": 0.0012129 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_00c8cee98bc3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1504, "completion_tokens": 193, "usd": 0.0006249 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_05517158aa56", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1567, "completion_tokens": 219, "usd": 0.0006672 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_bdbe9648c85c", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1556, "completion_tokens": 219, "usd": 0.0006639 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0131301, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.043767, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0131301, "ultra": 0.001013 }, "prompt_tokens_by_tier": { "nano": 0, "super": 33777, "ultra": 512 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 11.088, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-1", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_ac37081bc06f", "verdict_id": "verdict_ac37081bc06f", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would close issues in 5 of 13 replays.", "narrative": "Blocked: the new github-triage would close issues in 5 of 13 replays. Ratchet replayed 13 triage scenarios against the edited skill and detected deterministic side-effect regressions in five cases. Each regression involved new calls to github.close_issue, and sometimes github.comment, that the original runs never made; the sandbox refused those calls. The other eight replays passed with no drift or improvement. The edit changed only SKILL.md. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_f8c354ae33e9", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1604, "completion_tokens": 183, "usd": 0.0006459 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_478d9ee2ce19", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2670, "completion_tokens": 207, "usd": 0.0009873 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_755deee2dac3", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ce4b9acb3157", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 3 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 3, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_11ee7c7f4f30", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_9432a20fd19d", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3356, "completion_tokens": 334, "usd": 0.0013074 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_715dfb3b7eec", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_5ba0f0c4e92d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_39066d273f98", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3529, "completion_tokens": 309, "usd": 0.0013368 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_c944d9078bcc", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3307, "completion_tokens": 302, "usd": 0.0012639 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_56c2b63686fb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1587, "completion_tokens": 247, "usd": 0.0006984 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a8e17ca9255f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1614, "completion_tokens": 247, "usd": 0.0007065 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_dd5897c02420", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1639, "completion_tokens": 244, "usd": 0.0007113 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0076575, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.025525, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0076575, "ultra": 0.000787 }, "prompt_tokens_by_tier": { "nano": 0, "super": 19306, "ultra": 487 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 8.593, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_4360f846d2eb", "verdict_id": "verdict_4360f846d2eb", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The Tavily judge detected a regression in the \"Convert at a weekend date\" replay where the conversion rate shifted from 1.0768 to 1.0800, causing the expected date assertion to fail. Twelve other replays passed. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_885b30526626", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2511, "completion_tokens": 315, "usd": 0.0010368 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_fcf301073639", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "uses the correct date 2026-10-11 is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2744, "completion_tokens": 285, "usd": 0.0010797 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_82a70ccb2af8", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2587, "completion_tokens": 303, "usd": 0.0010488 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_973ca2670ff2", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2757, "completion_tokens": 306, "usd": 0.0011025 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3afca948a379", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2596, "completion_tokens": 341, "usd": 0.0010857 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_f5be008e5f9d", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2774, "completion_tokens": 313, "usd": 0.0011139 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_b55b161ede07", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2733, "completion_tokens": 354, "usd": 0.0011385 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_e70094e8923d", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2560, "completion_tokens": 312, "usd": 0.0010488 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5bde00727c59", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3052, "completion_tokens": 307, "usd": 0.0011919 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_96be56214418", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2745, "completion_tokens": 316, "usd": 0.0011079 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d1a68d58b4ae", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1694, "completion_tokens": 222, "usd": 0.000708 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_96c18d79778a", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 1.0, "prompt_tokens": 1697, "completion_tokens": 222, "usd": 0.0007089 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9cb3c3858cd1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1722, "completion_tokens": 212, "usd": 0.0007074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0130788, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.043596, "spent_usd_by_tier": { "nano": 0.00005004, "super": 0.0132696, "ultra": 0.000557 }, "prompt_tokens_by_tier": { "nano": 702, "super": 32757, "ultra": 305 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.41, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-1", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_7e1272baba99", "verdict_id": "verdict_7e1272baba99", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system ran 13 test replays against the proposed change and found 3 regressions. One replay produced an unauthorized github.comment call that the sandbox blocked. Two other replays failed to meet required output constraints on label counts and text formatting. Ten replays passed. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_165175858316", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1536, "completion_tokens": 179, "usd": 0.0006219 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_187d5cdcf6dc", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "contains no additional text that is not part of the required output format is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2588, "completion_tokens": 239, "usd": 0.0009915 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3f0a0fcb0075", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3264, "completion_tokens": 298, "usd": 0.0012474 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_9972d203f927", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3266, "completion_tokens": 290, "usd": 0.0012408 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c71e5c2370b8", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3040, "completion_tokens": 280, "usd": 0.001164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_eb9d9e6851a5", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "specifies that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3124, "completion_tokens": 299, "usd": 0.0012063 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_de7759ea9e17", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_092b5ebe7778", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3059, "completion_tokens": 295, "usd": 0.0011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_968755dee57c", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3642, "completion_tokens": 296, "usd": 0.001359 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_10dca42d0cf4", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3387, "completion_tokens": 299, "usd": 0.0012852 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_990c76131fc0", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1519, "completion_tokens": 198, "usd": 0.0006339 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_47c34378e66c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1558, "completion_tokens": 201, "usd": 0.0006483 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_602f5ee8bbac", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1583, "completion_tokens": 213, "usd": 0.0006666 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0122481, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.040827, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0122481, "ultra": 0.000595 }, "prompt_tokens_by_tier": { "nano": 0, "super": 31566, "ultra": 349 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 6.918, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_3f34abe4bb2c", "verdict_id": "verdict_3f34abe4bb2c", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 14 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 14 replays. Review before merging. The Tavily judge evaluated the fx-expense-converter skill against 14 replay pairs. Twelve passed, but two regressed. The \"Convert at a weekend date\" test failed because the exchange rate shifted from 1.0768 to 1.0800. The \"Rewrite a sentence formally\" test failed because the output now includes extra content beyond the formal statement. The change cost $0.0001 on nano and $0.01 on super. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_9b9fe0986841", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2331, "completion_tokens": 301, "usd": 0.0009702 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_f97cae02fbc4", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "uses the correct date 2026-10-11 is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2565, "completion_tokens": 287, "usd": 0.0010278 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6eb01a2d41d3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2416, "completion_tokens": 271, "usd": 0.0009687 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_315104ec0683", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2510, "completion_tokens": 292, "usd": 0.0010158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_762ca612dbe8", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2364, "completion_tokens": 308, "usd": 0.0009864 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_4ef2ca8f8b72", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2552, "completion_tokens": 278, "usd": 0.0010158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_53e1872e54f8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2576, "completion_tokens": 330, "usd": 0.0010698 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_fddd61f372fe", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2369, "completion_tokens": 289, "usd": 0.0009708 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_bda2112c7543", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2867, "completion_tokens": 276, "usd": 0.0011085 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c3051db65b47", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2543, "completion_tokens": 314, "usd": 0.0010455 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_8981ccff882f", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3828, "completion_tokens": 156, "usd": 0.0012888 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_ec291cce2b97", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "output must not include any additional content beyond the formal statement is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 1512, "completion_tokens": 220, "usd": 0.0006516 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_cbb67ab2491c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1519, "completion_tokens": 218, "usd": 0.0006519 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_91fcb24262ed", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1532, "completion_tokens": 211, "usd": 0.0006495 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0134211, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.044737, "spent_usd_by_tier": { "nano": 0.00005004, "super": 0.013611, "ultra": 0.000734 }, "prompt_tokens_by_tier": { "nano": 702, "super": 34072, "ultra": 338 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.067, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_430c14b02279", "verdict_id": "verdict_430c14b02279", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system evaluated the fx-expense-converter skill against 13 test cases, finding 12 pass and 1 regression. The regression occurred on the \"Compute 17% of 240\" test where the output no longer includes the expected phrase '= **40.8**'. This single failure triggered the hold despite the majority of tests passing. The change cost $0.01 to evaluate. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_382f2e4d44ae", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2390, "completion_tokens": 332, "usd": 0.0010158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_75e2cc6a76a8", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2602, "completion_tokens": 292, "usd": 0.0010434 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_c536f0cd0389", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2484, "completion_tokens": 277, "usd": 0.0009945 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_a8c493d9890b", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2593, "completion_tokens": 302, "usd": 0.0010497 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_b7a6a78b225c", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2423, "completion_tokens": 320, "usd": 0.0010149 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a00a986b53f7", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2614, "completion_tokens": 297, "usd": 0.0010515 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_c7bcfb7bf68e", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2645, "completion_tokens": 324, "usd": 0.0010851 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_3b832d8b5c25", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2428, "completion_tokens": 305, "usd": 0.0010029 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_f76c8a253ef0", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2854, "completion_tokens": 305, "usd": 0.0011307 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c432f688c91f", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2626, "completion_tokens": 322, "usd": 0.0010776 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1f1fa89dddfb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1563, "completion_tokens": 193, "usd": 0.0006426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_9470f1a93d4f", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "output includes the phrase '= **40.8**' is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1614, "completion_tokens": 219, "usd": 0.0006813 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f86b4f70f4f6", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1627, "completion_tokens": 216, "usd": 0.0006825 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0124725, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.041575, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0124725, "ultra": 0.000629 }, "prompt_tokens_by_tier": { "nano": 0, "super": 30463, "ultra": 275 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 11.984, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-1", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_549acf698c26", "verdict_id": "verdict_549acf698c26", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 22 replays. Review before merging. The system ran 22 replay pairs against the edited expense-report skill and found 21 passing with one regression in step m004. The regression shows the cleaned support/tickets.csv file no longer contains exactly three rows as expected. This indicates the recent change to SKILL.md likely altered the cleaning logic or filtering criteria for that specific dataset. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_ddb84b0fe47b", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1646, "completion_tokens": 141, "usd": 0.0006207 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_7d67f2353f20", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3141, "completion_tokens": 403, "usd": 0.001305 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_acc26084d722", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3981, "completion_tokens": 163, "usd": 0.001341 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_af5ade88f377", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2504, "completion_tokens": 283, "usd": 0.0010059 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_a69ed051d7f4", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3265, "completion_tokens": 304, "usd": 0.0012531 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_3e7d7797755d", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3214, "completion_tokens": 301, "usd": 0.0012351 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_079c36312676", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2529, "completion_tokens": 279, "usd": 0.0010098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_f07720304e5f", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2554, "completion_tokens": 288, "usd": 0.0010254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_8626f6fe5abb", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3388, "completion_tokens": 367, "usd": 0.0013467 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_521e59e6091f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3268, "completion_tokens": 321, "usd": 0.0012693 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_e31167be7710", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2344, "completion_tokens": 254, "usd": 0.0009318 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_229e9b0e57cc", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2698, "completion_tokens": 279, "usd": 0.0010605 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_1aec4a330ef6", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2559, "completion_tokens": 231, "usd": 0.0009756 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_5b8721009163", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2534, "completion_tokens": 273, "usd": 0.0010059 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3aee4f6f9c1d", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "the cleaned file support/tickets.clean.csv contains exactly 3 rows is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2728, "completion_tokens": 277, "usd": 0.0010677 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_838608cb3f0e", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2691, "completion_tokens": 302, "usd": 0.0010791 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_010f6ae26a1d", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2262, "completion_tokens": 253, "usd": 0.0009063 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_a8ba615a5d23", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2684, "completion_tokens": 286, "usd": 0.0010626 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_22e0e0ff85d9", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2736, "completion_tokens": 306, "usd": 0.0010962 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_15046a030360", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1580, "completion_tokens": 201, "usd": 0.0006549 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3ac334e2f481", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 1607, "completion_tokens": 242, "usd": 0.0006999 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb845c98bd90", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1632, "completion_tokens": 210, "usd": 0.0006786 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0226311, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.075437, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0226311, "ultra": 0.00057 }, "prompt_tokens_by_tier": { "nano": 0, "super": 57545, "ultra": 264 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.042, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-2", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_0be977370e6b", "verdict_id": "verdict_0be977370e6b", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system ran 13 regression pairs against the proposed change to SKILL.md and found 3 regressions. One case added an unexpected github.comment call that the sandbox rejected. Two other cases lost required label changes and a request for details on a specific issue. All 10 remaining pairs passed. The edit altered one line in the skill file. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5bbf428edf52", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1568, "completion_tokens": 186, "usd": 0.0006378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f2d56880f7aa", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2643, "completion_tokens": 213, "usd": 0.0009846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_c651c351049c", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3276, "completion_tokens": 284, "usd": 0.0012384 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_b6ff0cb2781b", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac63d98c8094", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3066, "completion_tokens": 305, "usd": 0.0011943 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_537a1694bbb0", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "specifies that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3358, "completion_tokens": 295, "usd": 0.0012729 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_9e2fa3314653", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d200a62ffe51", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3102, "completion_tokens": 289, "usd": 0.0011907 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_465ab5c43f9c", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "mentions that issue #36 was asked for details is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3391, "completion_tokens": 359, "usd": 0.0013404 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_19dcfee12b9c", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3240, "completion_tokens": 345, "usd": 0.0012825 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f59a42a6f25f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1549, "completion_tokens": 193, "usd": 0.0006384 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f5acfa748023", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1612, "completion_tokens": 223, "usd": 0.0006843 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_de28baa02c6f", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1625, "completion_tokens": 210, "usd": 0.0006765 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0111408, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.037136, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0111408, "ultra": 0.000637 }, "prompt_tokens_by_tier": { "nano": 0, "super": 28430, "ultra": 343 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.155, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-expense-report-1", "skill": "expense-report", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_1c7020d10d3d", "verdict_id": "verdict_1c7020d10d3d", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 7 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 7 of 22 replays. Review before merging. The system checked 22 test pairs against the edited expense-report skill, resulting in 15 passes and 7 regressions with no drift or improvements. Every regression shows the agent fails to extract merchant, date, total, and VAT data across diverse receipt types including bakery, software, dining, parking, office supplies, conference, lunch, train, taxi, and hotel expenses. The failures consistently occur at extraction step m003 or m005, indicating a systematic parsing breakdown rather than isolated cases. The edit changed the skill file with one addition and two deletions. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_73fd927ba82e", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1664, "completion_tokens": 151, "usd": 0.0006351 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_71599b6cf2f5", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3369, "completion_tokens": 289, "usd": 0.0012708 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_68029dbf8d42", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3824, "completion_tokens": 408, "usd": 0.0015144 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_0061479721a1", "task_id": "exp-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the VAT 18.81 EUR is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2522, "completion_tokens": 289, "usd": 0.0010167 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_cde84ba8b5d5", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 3211, "completion_tokens": 319, "usd": 0.0012504 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_d2bfe3e90095", "task_id": "exp-dinner-parking", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3214, "completion_tokens": 325, "usd": 0.0012567 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_ea8d1c2b4f4b", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the VAT amount '7.58 EUR' is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2495, "completion_tokens": 268, "usd": 0.0009897 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_bcff8a71d40d", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the VAT as 79.83 EUR is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2535, "completion_tokens": 286, "usd": 0.0010179 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_49a1a41ef5ee", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3380, "completion_tokens": 391, "usd": 0.0013659 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_87ffd7a9b61f", "task_id": "exp-berlin-trip", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3208, "completion_tokens": 327, "usd": 0.0012567 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_bfab509e3676", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2412, "completion_tokens": 263, "usd": 0.0009603 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_ae36c7d648f7", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2724, "completion_tokens": 339, "usd": 0.0011223 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_c94efe16dd1d", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2489, "completion_tokens": 242, "usd": 0.0009645 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ffd4fd935f19", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2492, "completion_tokens": 272, "usd": 0.0009924 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_699475671d46", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2772, "completion_tokens": 309, "usd": 0.0011097 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_bbc5ac91d665", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2740, "completion_tokens": 316, "usd": 0.0011064 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_657e87d6ef5e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2232, "completion_tokens": 273, "usd": 0.0009153 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_8d54d0935d9d", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2588, "completion_tokens": 304, "usd": 0.00105 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_cbe81857d988", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2600, "completion_tokens": 342, "usd": 0.0010878 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_db46f3b0746f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1574, "completion_tokens": 221, "usd": 0.0006711 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_10a2feb40124", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1613, "completion_tokens": 229, "usd": 0.00069 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_da322788c4ba", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1626, "completion_tokens": 217, "usd": 0.0006831 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0229272, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.076424, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0229272, "ultra": 0.000941 }, "prompt_tokens_by_tier": { "nano": 0, "super": 57284, "ultra": 494 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.418, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-2", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_eb4f4937c7bb", "verdict_id": "verdict_eb4f4937c7bb", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. Ratchet replayed 13 past tasks against the edit to github-triage and compared each with its blessed result. On \"Triage acme/status-page (all labeled)\": Got worse. contains no additional text that is not part of the required output format is no longer met. (step m003) On \"Triage acme/mobile-app\": Got worse. labels #58 as bug is no longer met. (step t002) Judging cost $0.01 (super). Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0a31fbd0f6ab", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1537, "completion_tokens": 180, "usd": 0.0006231 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e3ade3806218", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "contains no additional text that is not part of the required output format is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2562, "completion_tokens": 231, "usd": 0.0009765 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e7ec930d750a", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3185, "completion_tokens": 257, "usd": 0.0011868 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_c0c9ec6a7dc8", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3279, "completion_tokens": 261, "usd": 0.0012186 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_9484d74e29ab", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3021, "completion_tokens": 252, "usd": 0.0011331 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_dcfc05cb8a6e", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 3135, "completion_tokens": 306, "usd": 0.0012159 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_b752b4bba738", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3017, "completion_tokens": 258, "usd": 0.0011373 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_811329a74d4f", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "labels #58 as bug is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2995, "completion_tokens": 307, "usd": 0.0011748 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9dd1c1c9ae58", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3635, "completion_tokens": 295, "usd": 0.001356 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d9834ffa3e3a", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3330, "completion_tokens": 296, "usd": 0.0012654 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3f0ff18aec4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1472, "completion_tokens": 217, "usd": 0.0006369 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_5b3dba8d5830", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1523, "completion_tokens": 219, "usd": 0.000654 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f28450be2e37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1536, "completion_tokens": 212, "usd": 0.0006516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.01323, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.0441, "spent_usd_by_tier": { "nano": 0.0, "super": 0.01323, "ultra": 0.000608 }, "prompt_tokens_by_tier": { "nano": 0, "super": 34227, "ultra": 302 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 11.108, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_c7cc8cadc258", "verdict_id": "verdict_c7cc8cadc258", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated skill and every pair passed with zero regressions, drift, or uncertain results. The change modified only the SKILL.md documentation file with a single line edit, leaving the core conversion logic untouched. Cost was negligible at one cent. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_bf3fa0074e07", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2303, "completion_tokens": 299, "usd": 0.00096 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_e649669b0a50", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2567, "completion_tokens": 269, "usd": 0.0010122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6b67a590cae3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2426, "completion_tokens": 284, "usd": 0.0009834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_1f156a1c8f92", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2508, "completion_tokens": 296, "usd": 0.0010188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3e98cef87a9d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2350, "completion_tokens": 319, "usd": 0.0009921 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9ffedc5fd087", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2566, "completion_tokens": 303, "usd": 0.0010425 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_941456f71c6b", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2550, "completion_tokens": 352, "usd": 0.0010818 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_4edc319d8099", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2355, "completion_tokens": 312, "usd": 0.0009873 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5ece1e8c9118", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2768, "completion_tokens": 320, "usd": 0.0011184 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_400354316051", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2468, "completion_tokens": 312, "usd": 0.0010212 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_6cfa1d232eb4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1490, "completion_tokens": 193, "usd": 0.0006207 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_13feb5997c98", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1529, "completion_tokens": 209, "usd": 0.0006468 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e2f6c6c99b4", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1542, "completion_tokens": 211, "usd": 0.0006525 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0121377, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.040459, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0121377, "ultra": 0.000469 }, "prompt_tokens_by_tier": { "nano": 0, "super": 29422, "ultra": 229 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 12.756, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-2", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_c41435ee341f", "verdict_id": "verdict_c41435ee341f", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system evaluated the csv-cleaner skill change against 13 test cases; 12 passed but one regression was detected where the output for \"Compute 17% of 240\" no longer includes the expected phrase '= **40.8**'. The evaluation cost $0.01. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_959e7e0816aa", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2237, "completion_tokens": 295, "usd": 0.0009366 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_b81786a373a5", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2408, "completion_tokens": 296, "usd": 0.0009888 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_1dca9722e993", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2347, "completion_tokens": 269, "usd": 0.0009462 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_96dcc8778ae9", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2790, "completion_tokens": 283, "usd": 0.0010917 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_130fa3316eae", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2708, "completion_tokens": 295, "usd": 0.0010779 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_b7dba255cdef", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2551, "completion_tokens": 314, "usd": 0.0010479 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_fe6d06e8fe47", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2682, "completion_tokens": 283, "usd": 0.0010593 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_5d23e89670d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2604, "completion_tokens": 325, "usd": 0.0010737 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_02fdaa1af68e", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3356, "completion_tokens": 271, "usd": 0.0012507 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7c626553532c", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2593, "completion_tokens": 271, "usd": 0.0010218 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8b65fb6e74ff", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1504, "completion_tokens": 212, "usd": 0.000642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f33745e12a8e", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "output includes the phrase '= **40.8**' is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1543, "completion_tokens": 219, "usd": 0.00066 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_690661728b24", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1556, "completion_tokens": 231, "usd": 0.0006747 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0124713, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.041571, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0124713, "ultra": 0.000564 }, "prompt_tokens_by_tier": { "nano": 0, "super": 30879, "ultra": 273 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.862, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-2", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_4218db7135a9", "verdict_id": "verdict_4218db7135a9", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 6 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 6 of 13 replays. The system compared the edited skill against the original across 13 recorded task runs. Six replays produced new github.comment calls that the original runs never made; the sandbox rejected those calls, marking the pairs as regressions. The remaining seven replays passed with identical behavior. The change introduces unauthorized commenting side effects across multiple repositories. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_aeb36d32b26b", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1677, "completion_tokens": 180, "usd": 0.0006651 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_fb3621bc1546", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2706, "completion_tokens": 220, "usd": 0.0010098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_a026774bb5dc", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3351, "completion_tokens": 309, "usd": 0.0012834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_082c7d58917e", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_e82538fe8409", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_4281d13e3b01", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7ee8c01d51f9", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4f8712b7b45d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_650cb16fa7da", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_6f78304e2ae4", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_96b50023c3c8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1612, "completion_tokens": 217, "usd": 0.0006789 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_575a1d924d31", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1699, "completion_tokens": 201, "usd": 0.0006906 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8419e2ca5496", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1712, "completion_tokens": 210, "usd": 0.0007026 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0050304, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.016768, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0050304, "ultra": 0.000797 }, "prompt_tokens_by_tier": { "nano": 0, "super": 12757, "ultra": 518 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 7.735, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-expense-report-1", "skill": "expense-report", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_55b447ab49d0", "verdict_id": "verdict_55b447ab49d0", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The single-line change to SKILL.md was evaluated against the full regression suite of 22 task pairs. Every pair produced the expected output with zero regressions, zero drift, and zero uncertain results. The edit cost $0.02 in super compute. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_60c12e465a52", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1611, "completion_tokens": 161, "usd": 0.0006282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_87b3a1580656", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3096, "completion_tokens": 338, "usd": 0.001233 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_ac01bb0f765a", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3828, "completion_tokens": 419, "usd": 0.0015255 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_ba09cf46ac3d", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2435, "completion_tokens": 281, "usd": 0.0009834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_ca6090d7b4f3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3163, "completion_tokens": 325, "usd": 0.0012414 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_f88f40b0fdd4", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3121, "completion_tokens": 329, "usd": 0.0012324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_4c04fd872a46", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2484, "completion_tokens": 307, "usd": 0.0010215 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a5beee38284e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2500, "completion_tokens": 323, "usd": 0.0010407 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37245d26f903", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3330, "completion_tokens": 400, "usd": 0.001359 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_4e2b952bb502", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3163, "completion_tokens": 307, "usd": 0.0012252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_803282d49f8b", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2298, "completion_tokens": 257, "usd": 0.0009207 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_c5f95c50e3d5", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2564, "completion_tokens": 263, "usd": 0.0010059 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_8a5a0ea0ed7f", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2548, "completion_tokens": 325, "usd": 0.0010569 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_812811600cab", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2465, "completion_tokens": 257, "usd": 0.0009708 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_eb78878a2ed4", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2865, "completion_tokens": 277, "usd": 0.0011088 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0932e89999f7", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2706, "completion_tokens": 299, "usd": 0.0010809 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_154346b1b35e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2215, "completion_tokens": 277, "usd": 0.0009138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_388513c72ff0", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2827, "completion_tokens": 295, "usd": 0.0011136 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_1d8ff02cb8d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2637, "completion_tokens": 312, "usd": 0.0010719 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_43b809e3abdd", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1523, "completion_tokens": 218, "usd": 0.0006531 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_62160dffa87e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1538, "completion_tokens": 201, "usd": 0.0006423 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f2e5587a73ca", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1565, "completion_tokens": 220, "usd": 0.0006675 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0226965, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.075655, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0226965, "ultra": 0.000436 }, "prompt_tokens_by_tier": { "nano": 0, "super": 56482, "ultra": 220 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 33.906, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-2", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_01fd96293b51", "verdict_id": "verdict_01fd96293b51", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The system ran 13 regression pairs against the updated github-triage skill. Eleven passed, but two failed. In the \"Triage acme/search-indexer\" replay, the agent no longer labels issue #502 as a question. In the \"Compute 17% of 240\" replay, the expected output \"= **40.8**\" is missing. The test cost $0.01. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5da690453eaf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1561, "completion_tokens": 180, "usd": 0.0006303 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_66be25a1b831", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2389, "completion_tokens": 205, "usd": 0.0009012 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_9803f74fad28", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "labels issue #502 as question is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3259, "completion_tokens": 305, "usd": 0.0012522 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_287402e48fdd", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3227, "completion_tokens": 261, "usd": 0.001203 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ad012c68b2ce", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3025, "completion_tokens": 244, "usd": 0.0011271 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_c31ee7fd6635", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3124, "completion_tokens": 311, "usd": 0.0012171 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_a3e558951509", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3033, "completion_tokens": 331, "usd": 0.0012078 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7af9bb18724a", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3237, "completion_tokens": 305, "usd": 0.0012456 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9ea5f7e643be", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3431, "completion_tokens": 294, "usd": 0.0012939 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_acc745f38e82", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3152, "completion_tokens": 292, "usd": 0.0012084 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3ac050839ba", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1544, "completion_tokens": 212, "usd": 0.000654 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1e223b9eccf6", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "output includes the phrase '= **40.8**' is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 1571, "completion_tokens": 218, "usd": 0.0006675 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a03ddffaa752", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1596, "completion_tokens": 218, "usd": 0.000675 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0132831, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.044277, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0132831, "ultra": 0.000664 }, "prompt_tokens_by_tier": { "nano": 0, "super": 34149, "ultra": 304 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 10.648, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-1", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_d39145edd02a", "verdict_id": "verdict_d39145edd02a", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The system replayed 13 test cases against the edited github-triage skill and detected two regressions. In \"Triage acme/data-pipeline,\" the agent no longer produces exactly six label changes at step t005. In \"Triage acme/billing-api,\" the agent no longer reports the correct number of open backlog issues at step t004. The other 11 cases passed. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_bd63d6f067f1", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1617, "completion_tokens": 188, "usd": 0.0006543 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f6019126d256", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2647, "completion_tokens": 216, "usd": 0.0009885 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_dd87261b8ba6", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3361, "completion_tokens": 294, "usd": 0.0012729 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_2d58c0602c67", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3354, "completion_tokens": 344, "usd": 0.0013158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c9de1093aa99", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3091, "completion_tokens": 285, "usd": 0.0011838 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5101ae1012ad", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "specifies that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3391, "completion_tokens": 303, "usd": 0.00129 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_47ce9e153501", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3270, "completion_tokens": 304, "usd": 0.0012546 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4db894881c54", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3115, "completion_tokens": 292, "usd": 0.0011973 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_c8c4828aae44", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "states the number of open issues in the acme/billing-api backlog is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3510, "completion_tokens": 346, "usd": 0.0013644 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_b3c44e403f10", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3242, "completion_tokens": 356, "usd": 0.001293 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f3095a10d7d9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1588, "completion_tokens": 223, "usd": 0.0006771 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_4519fd683ef9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1651, "completion_tokens": 219, "usd": 0.0006924 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_72d1ef164d37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1628, "completion_tokens": 211, "usd": 0.0006783 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0138624, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.046208, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0138624, "ultra": 0.000654 }, "prompt_tokens_by_tier": { "nano": 0, "super": 35465, "ultra": 303 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 12.801, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_5df39fa2d2ee", "verdict_id": "verdict_5df39fa2d2ee", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The Tavily judge detected a regression in the \"Convert at a weekend date\" replay where the conversion rate shifted from 1.0768 to 1.0800, causing the expected date assertion to fail. Twelve other replays passed. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e5b902ded754", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2340, "completion_tokens": 291, "usd": 0.0009639 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_8c4cb14a441e", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "uses the correct date 2026-10-11 is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2649, "completion_tokens": 284, "usd": 0.0010503 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_3f28b051e158", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2475, "completion_tokens": 277, "usd": 0.0009918 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_bdb089b8bdbb", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2700, "completion_tokens": 306, "usd": 0.0010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_2393bfd5f040", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2434, "completion_tokens": 330, "usd": 0.0010272 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a029bb9d7e18", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2653, "completion_tokens": 285, "usd": 0.0010524 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_148da7df7bf5", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2657, "completion_tokens": 357, "usd": 0.0011184 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_c24c16f3b5fa", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2462, "completion_tokens": 300, "usd": 0.0010086 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_b4e1ea78a932", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2882, "completion_tokens": 298, "usd": 0.0011328 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_cd2b69e99407", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2638, "completion_tokens": 317, "usd": 0.0010767 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_83397ea2c263", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1585, "completion_tokens": 221, "usd": 0.0006744 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_37f7a8b8bf4c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1612, "completion_tokens": 221, "usd": 0.0006825 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8108d03dbe59", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1639, "completion_tokens": 220, "usd": 0.0006897 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0125541, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.041847, "spent_usd_by_tier": { "nano": 0.00005028, "super": 0.012744, "ultra": 0.000557 }, "prompt_tokens_by_tier": { "nano": 702, "super": 31314, "ultra": 305 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 17.217, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-1", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_a2561bccd378", "verdict_id": "verdict_a2561bccd378", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The system replayed 13 test cases against the proposed change to github-triage. Twelve passed, but the \"Triage acme/data-pipeline\" case regressed at step t005 because the agent no longer produces exactly six github.label changes. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_36523cbf4c92", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1548, "completion_tokens": 169, "usd": 0.0006165 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e70cb39d820f", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2508, "completion_tokens": 244, "usd": 0.000972 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_93bcdbcd3914", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3222, "completion_tokens": 263, "usd": 0.0012033 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ba25466de412", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3254, "completion_tokens": 265, "usd": 0.0012147 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_eb462443ad9b", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2991, "completion_tokens": 277, "usd": 0.0011466 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5d5629ab1937", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "specifies that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3126, "completion_tokens": 316, "usd": 0.0012222 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_37a737ab09a1", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2958, "completion_tokens": 313, "usd": 0.0011691 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a8f42769e490", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3023, "completion_tokens": 286, "usd": 0.0011643 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dbbba87c46a3", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3625, "completion_tokens": 315, "usd": 0.001371 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_84d120ce3156", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3164, "completion_tokens": 327, "usd": 0.0012435 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f1c02a8e442b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1495, "completion_tokens": 193, "usd": 0.0006222 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_227ef3c2f411", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1558, "completion_tokens": 219, "usd": 0.0006645 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_ed4fe7d3e21d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1583, "completion_tokens": 228, "usd": 0.0006801 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.01329, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.0443, "spent_usd_by_tier": { "nano": 0.0, "super": 0.01329, "ultra": 0.000515 }, "prompt_tokens_by_tier": { "nano": 0, "super": 34055, "ultra": 263 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 7.98, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_ee93a705cfb8", "verdict_id": "verdict_ee93a705cfb8", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated skill and every pair passed with zero regressions, drift, or uncertain results. The change modified only the SKILL.md file with a minimal diff of one line added and one removed. The cost was negligible at one cent. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_8c2c5e26d4e1", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2399, "completion_tokens": 303, "usd": 0.0009924 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_2a31e3e43325", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2705, "completion_tokens": 304, "usd": 0.0010851 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_40498db1a5f2", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2601, "completion_tokens": 264, "usd": 0.0010179 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_0f0b98c0dda1", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3023, "completion_tokens": 315, "usd": 0.0011904 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_a341fb0948fa", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2530, "completion_tokens": 329, "usd": 0.0010551 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_b5a453299350", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2704, "completion_tokens": 269, "usd": 0.0010533 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ea5e1042bbf8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2690, "completion_tokens": 336, "usd": 0.0011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ab59cb62584a", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2473, "completion_tokens": 303, "usd": 0.0010146 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_d101d55224e4", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2925, "completion_tokens": 296, "usd": 0.0011439 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_f96f5f3cef70", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2702, "completion_tokens": 289, "usd": 0.0010707 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_eb8645af7d9f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1632, "completion_tokens": 221, "usd": 0.0006885 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_710563b99aef", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 1647, "completion_tokens": 220, "usd": 0.0006921 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb5c11b812f5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1686, "completion_tokens": 220, "usd": 0.0007038 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0128172, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.042724, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0128172, "ultra": 0.000463 }, "prompt_tokens_by_tier": { "nano": 0, "super": 31717, "ultra": 229 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.058, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-2", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_6cdc06629a08", "verdict_id": "verdict_6cdc06629a08", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 22 replays. Review before merging. The system ran 22 test pairs against the edited expense-report skill, resulting in 21 passes and 1 regression. The failure occurred on the \"File a bakery receipt and a missing one\" case where the agent no longer states the merchant, date, total, and VAT for both receipts at step m004. The change cost $0.02. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_efb6b4383a27", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1873, "completion_tokens": 175, "usd": 0.0007194 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_26a650f173a0", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3434, "completion_tokens": 315, "usd": 0.0013137 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_90c84e6db9bc", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3976, "completion_tokens": 436, "usd": 0.0015852 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_9f99d91e21f6", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2721, "completion_tokens": 267, "usd": 0.0010566 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_29cce0ac5de3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3394, "completion_tokens": 303, "usd": 0.0012909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_21bf7c25b237", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3391, "completion_tokens": 317, "usd": 0.0013026 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_256b30ee0efe", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2722, "completion_tokens": 297, "usd": 0.0010839 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_dfea8f3d0de4", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2753, "completion_tokens": 273, "usd": 0.0010716 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_f1ddb4f3653c", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3738, "completion_tokens": 351, "usd": 0.0014373 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_ef0acc9e5a05", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3442, "completion_tokens": 310, "usd": 0.0013116 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_11286bfe394d", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2557, "completion_tokens": 269, "usd": 0.0010092 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_325c7d095b64", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2997, "completion_tokens": 282, "usd": 0.0011529 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_2a529632a34a", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2745, "completion_tokens": 224, "usd": 0.0010251 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_908da659aa18", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2750, "completion_tokens": 328, "usd": 0.0011202 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3a7988db6226", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2953, "completion_tokens": 268, "usd": 0.0011271 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_3186365c17c2", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2944, "completion_tokens": 302, "usd": 0.001155 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_780ead4737e0", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2443, "completion_tokens": 267, "usd": 0.0009732 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_0ba4c66e04ad", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2783, "completion_tokens": 310, "usd": 0.0011139 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_58c7d9abe244", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2810, "completion_tokens": 297, "usd": 0.0011103 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8e9bcc22b033", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1781, "completion_tokens": 218, "usd": 0.0007305 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1cc2a5f894e5", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 1820, "completion_tokens": 218, "usd": 0.0007422 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_07aa4d542d5a", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1833, "completion_tokens": 210, "usd": 0.0007389 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0241713, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.080571, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0241713, "ultra": 0.000599 }, "prompt_tokens_by_tier": { "nano": 0, "super": 61860, "ultra": 272 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 17.878, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_00ebfccf372f", "verdict_id": "verdict_00ebfccf372f", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The Tavily judge detected a regression in the \"Convert at a weekend date\" replay where the conversion rate shifted from 1.0768 to 1.0800, causing the expected date assertion to fail. Twelve other replays passed. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_88832e95c9b9", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2433, "completion_tokens": 317, "usd": 0.0010152 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_33a9c6adf157", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "uses the correct date 2026-10-11 is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2671, "completion_tokens": 290, "usd": 0.0010623 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_07551b4c2872", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2527, "completion_tokens": 275, "usd": 0.0010056 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f725843c1f80", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2740, "completion_tokens": 289, "usd": 0.0010821 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_0545f282df0a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2516, "completion_tokens": 354, "usd": 0.0010734 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9df69746323e", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2680, "completion_tokens": 258, "usd": 0.0010362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_51bbbd84f8be", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2703, "completion_tokens": 330, "usd": 0.0011079 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_32ea0f012a38", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2461, "completion_tokens": 299, "usd": 0.0010074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_1847ee62554a", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2942, "completion_tokens": 302, "usd": 0.0011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_08c9424f7da6", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2659, "completion_tokens": 299, "usd": 0.0010668 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_a144e4814df4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1608, "completion_tokens": 221, "usd": 0.0006813 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e625a12fe4a6", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1683, "completion_tokens": 223, "usd": 0.0007056 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c818c2acd85b", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1650, "completion_tokens": 220, "usd": 0.000693 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0126912, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.042304, "spent_usd_by_tier": { "nano": 0.00005076, "super": 0.0128811, "ultra": 0.000557 }, "prompt_tokens_by_tier": { "nano": 702, "super": 31861, "ultra": 305 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.363, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_4e3bd0c56568", "verdict_id": "verdict_4e3bd0c56568", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated skill and every pair passed with zero regressions, drift, or uncertain results. The change modified only the SKILL.md documentation file with a single line edit, leaving the core conversion logic untouched. At a cost of one cent, the verification confirms the agent behaves identically to the previously blessed version across all known scenarios. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e9f830fa642e", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2615, "completion_tokens": 312, "usd": 0.0010653 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_bbae78eada7e", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2817, "completion_tokens": 297, "usd": 0.0011124 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_518252b4621f", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2764, "completion_tokens": 296, "usd": 0.0010956 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_6bd370a9ae4d", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2980, "completion_tokens": 355, "usd": 0.0012135 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_4a19b9d5037d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2716, "completion_tokens": 349, "usd": 0.0011289 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_659d9e0a0119", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2831, "completion_tokens": 332, "usd": 0.0011481 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ae6602b84c3d", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3034, "completion_tokens": 371, "usd": 0.0012441 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_76bd7f61f20c", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2708, "completion_tokens": 342, "usd": 0.0011202 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_7b5ee5b9fe63", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3124, "completion_tokens": 306, "usd": 0.0012126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_40cfb64bd6bb", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3084, "completion_tokens": 318, "usd": 0.0012114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2c596c557862", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1637, "completion_tokens": 223, "usd": 0.0006918 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_dd37987a313e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1676, "completion_tokens": 229, "usd": 0.0007089 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_14951b257b04", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1701, "completion_tokens": 210, "usd": 0.0006993 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0136521, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.045507, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0136521, "ultra": 0.000523 }, "prompt_tokens_by_tier": { "nano": 0, "super": 33687, "ultra": 229 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.461, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-2", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_8e02fe26d92f", "verdict_id": "verdict_8e02fe26d92f", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The edit changed github-triage SKILL.md with four additions and no deletions. Thirteen evaluation pairs ran: twelve passed, one was uncertain. The uncertain case, \"Triage without a repository,\" showed neither the golden nor the replay run completing a triage action; both only requested a repository specification. Because the judge could not verify the success criteria were met, the decision was held for human review. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0e10ad5bac34", "task_id": "tri-no-repo", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "super", "reason": "Insufficient information to determine if the success criteria are met because neither the golden nor replay runs show a completed triage action—only a request for repository specification. Held for a human.", "opinions": [ { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 1569, "completion_tokens": 72, "usd": 0.0005355 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e27ff37067f7", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2598, "completion_tokens": 206, "usd": 0.0009648 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3d9dd364ab27", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3275, "completion_tokens": 255, "usd": 0.001212 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_570420a9b31d", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3247, "completion_tokens": 282, "usd": 0.0012279 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_40e6b8dc80bd", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3015, "completion_tokens": 304, "usd": 0.0011781 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5a38101f8f14", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3135, "completion_tokens": 326, "usd": 0.0012339 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_1f166314792c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2999, "completion_tokens": 326, "usd": 0.0011931 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d9f90397db42", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3059, "completion_tokens": 279, "usd": 0.0011688 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_13d086a4d0a2", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3412, "completion_tokens": 334, "usd": 0.0013242 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_24cc0a76ffb7", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3368, "completion_tokens": 289, "usd": 0.0012705 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_86211c1cba50", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1518, "completion_tokens": 194, "usd": 0.00063 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_70087be29148", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 1581, "completion_tokens": 224, "usd": 0.0006759 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3b107f1aecf5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1594, "completion_tokens": 212, "usd": 0.000669 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0132837, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.044279, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0132837, "ultra": 0.000611 }, "prompt_tokens_by_tier": { "nano": 0, "super": 34370, "ultra": 278 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 11.838, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-2", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_2f5cc5064b44", "verdict_id": "verdict_2f5cc5064b44", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system replayed 13 test cases against the proposed change; 11 passed but 2 regressed. In the acme/infra-terraform replay, the agent made two new comment calls that the sandbox rejected. In the acme/data-pipeline replay, the agent failed to make the expected six label changes. The edit only modified SKILL.md. Roll back the change or fix the skill logic to stop the extra comments and restore the correct label behavior. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_9aae4e6186d2", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1534, "completion_tokens": 186, "usd": 0.0006276 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_852cf7299d83", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2589, "completion_tokens": 244, "usd": 0.0009963 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_1f4cd7269ecb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3259, "completion_tokens": 262, "usd": 0.0012135 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d6db457362fe", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_7d366fbfea40", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2968, "completion_tokens": 281, "usd": 0.0011433 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_bb40249ba200", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "specifies that exactly six github.label changes occurred is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3094, "completion_tokens": 332, "usd": 0.001227 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_5cb5a86b5b0c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3147, "completion_tokens": 267, "usd": 0.0011844 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_8e097e2bc153", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3007, "completion_tokens": 293, "usd": 0.0011658 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dc72a3a720da", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3427, "completion_tokens": 300, "usd": 0.0012981 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d4e29e63139d", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3363, "completion_tokens": 341, "usd": 0.0013158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1bcf329ed33b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1493, "completion_tokens": 194, "usd": 0.0006225 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_20de0884c5fb", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1544, "completion_tokens": 219, "usd": 0.0006603 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b123eb9839bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1557, "completion_tokens": 220, "usd": 0.0006651 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0121197, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.040399, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0121197, "ultra": 0.000676 }, "prompt_tokens_by_tier": { "nano": 0, "super": 30982, "ultra": 310 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.029, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_532d20f9e8fb", "verdict_id": "verdict_532d20f9e8fb", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 1 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 1 of 13 replays. Review before merging. The Tavily judge evaluated 13 replay pairs against the updated fx-expense-converter skill, finding 12 passing and 1 regression. The regression occurred in the \"Convert at a weekend date\" scenario where the expected rate of 1.0768 shifted to 1.0800 at step m003. The change cost $0.0001 for nano and $0.01 for super tiers. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_4e739b0fed3d", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2256, "completion_tokens": 304, "usd": 0.0009504 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_d7ede738861c", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "uses the correct date 2026-10-11 is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2543, "completion_tokens": 289, "usd": 0.001023 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_7f573055a9ef", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2381, "completion_tokens": 291, "usd": 0.0009762 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f4c1c98c1a21", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2620, "completion_tokens": 297, "usd": 0.0010533 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_6b7462d13be4", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2413, "completion_tokens": 345, "usd": 0.0010344 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_914ca7824ded", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2547, "completion_tokens": 258, "usd": 0.0009963 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_da9dc2ce34c7", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2552, "completion_tokens": 362, "usd": 0.0010914 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_70ade45e8144", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2476, "completion_tokens": 312, "usd": 0.0010236 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_0b662be246d7", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2832, "completion_tokens": 287, "usd": 0.0011079 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_ca8b25ae83a2", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2544, "completion_tokens": 298, "usd": 0.0010314 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_0bda149accdc", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1457, "completion_tokens": 212, "usd": 0.0006279 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e0d76dcf56dd", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1532, "completion_tokens": 220, "usd": 0.0006576 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f3b045e8f40e", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1557, "completion_tokens": 219, "usd": 0.0006642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0122376, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.040792, "spent_usd_by_tier": { "nano": 0.00005004, "super": 0.0124275, "ultra": 0.000677 }, "prompt_tokens_by_tier": { "nano": 702, "super": 30298, "ultra": 305 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.066, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-1", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_0f1d66559089", "verdict_id": "verdict_0f1d66559089", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 4 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 4 of 13 replays. Review before merging. The system re-ran 13 test cases against the edited github-triage skill and compared outputs using an LLM judge. Nine cases passed, but four regressed. The strongest failures involve the triage skill incorrectly handling already-labeled issues by adding extra text or duplicate lines that violate the required output format, and a math case failing because the output included a specific bolded phrase. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_fc1895f58dcf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1581, "completion_tokens": 187, "usd": 0.0006426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_99daafbd39ef", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "contains no additional text that is not part of the required output format is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2620, "completion_tokens": 250, "usd": 0.001011 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_204e00dc72de", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "labels issue #503 as already labeled is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3282, "completion_tokens": 338, "usd": 0.0012888 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d8383ef71ebb", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3316, "completion_tokens": 289, "usd": 0.0012549 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_f644709d32f1", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3045, "completion_tokens": 275, "usd": 0.001161 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_92e4b2c50b1c", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3354, "completion_tokens": 359, "usd": 0.0013293 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_574e79e6ac2e", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3020, "completion_tokens": 312, "usd": 0.0011868 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7213bfe2b9f8", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "The replay output includes extra lines '#62: bug (already labeled)' and duplicates 'Already labeled: #62', violating the required format which should have exactly one 'Already labeled:' line and no per-issue labeling statements for already… is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3070, "completion_tokens": 300, "usd": 0.001191 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_a05f5333a824", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3688, "completion_tokens": 296, "usd": 0.0013728 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_1891e80878db", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3241, "completion_tokens": 373, "usd": 0.001308 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d5662e056952", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1564, "completion_tokens": 244, "usd": 0.0006888 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e3d06ab590fa", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "output includes the phrase \"= **40.8**\" is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1579, "completion_tokens": 225, "usd": 0.0006762 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_18d4cf624d2d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1592, "completion_tokens": 213, "usd": 0.0006693 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0137805, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.045935, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0137805, "ultra": 0.000742 }, "prompt_tokens_by_tier": { "nano": 0, "super": 34952, "ultra": 418 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 11.849, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-expense-report-2", "skill": "expense-report", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_aeaab0d491a7", "verdict_id": "verdict_aeaab0d491a7", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 3 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 3 of 22 replays. Review before merging. The judge evaluated 22 test pairs and found 19 passing, 2 regressions, and 1 uncertain result. The first regression shows the skill no longer extracts merchant, date, total, and VAT for two receipts. The second shows a CSV cleaning step produces the wrong row count. The uncertain case involves a criteria mismatch where the test expects lead data but the replay runs expense steps. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2f14f21d6d9f", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1565, "completion_tokens": 176, "usd": 0.0006279 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_e92a7282295d", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3172, "completion_tokens": 327, "usd": 0.0012459 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_a6b930ea6aff", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 4031, "completion_tokens": 424, "usd": 0.0015909 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_e2914d406490", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2583, "completion_tokens": 297, "usd": 0.0010422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_07a85852522a", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3308, "completion_tokens": 289, "usd": 0.0012525 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_85a67c3ff72c", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3301, "completion_tokens": 305, "usd": 0.0012648 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_afcfe0768c5a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2616, "completion_tokens": 304, "usd": 0.0010584 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a64afbf6b2cb", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2623, "completion_tokens": 340, "usd": 0.0010929 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_9c6ec62f2eb9", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3486, "completion_tokens": 405, "usd": 0.0014103 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6ad0a889dbc3", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3295, "completion_tokens": 332, "usd": 0.0012873 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_b6e697e319df", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2501, "completion_tokens": 281, "usd": 0.0010032 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_8ffffbf26485", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2859, "completion_tokens": 270, "usd": 0.0011007 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_56a48c4930ba", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2622, "completion_tokens": 264, "usd": 0.0010242 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_4c8e63150ce8", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2658, "completion_tokens": 302, "usd": 0.0010692 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_aa8e42f0327e", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "the cleaned file support/tickets.clean.csv contains exactly 3 rows is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3373, "completion_tokens": 300, "usd": 0.0012819 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_c3600f8c1037", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2791, "completion_tokens": 311, "usd": 0.0011172 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_64e31b08b933", "task_id": "csv-header-only", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "super", "reason": "The success criteria refer to leads.clean.csv but the provided golden and replay runs concern expense-report skill steps on receipts.parse and sheets.append, showing no overlap. Held for a human.", "opinions": [ { "tier": "super", "verdict": "uncertain", "confidence": 0.3, "prompt_tokens": 2996, "completion_tokens": 93, "usd": 0.0009825 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_c2e995113d65", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2746, "completion_tokens": 304, "usd": 0.0010974 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_18bb09c7d462", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2757, "completion_tokens": 315, "usd": 0.0011106 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_de7f8d3695ad", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1655, "completion_tokens": 216, "usd": 0.0006909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_40e8a755ae4f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1694, "completion_tokens": 249, "usd": 0.0007323 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b58e85552abd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1707, "completion_tokens": 213, "usd": 0.0007038 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.023787, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.07929, "spent_usd_by_tier": { "nano": 0.0, "super": 0.023787, "ultra": 0.000699 }, "prompt_tokens_by_tier": { "nano": 0, "super": 60339, "ultra": 363 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.983, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-expense-report-1", "skill": "expense-report", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_0f1b7c262b45", "verdict_id": "verdict_0f1b7c262b45", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 5 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 5 of 22 replays. Review before merging. The system re-ran 22 test pairs against the edited expense-report skill, resulting in 17 passes and 5 regressions with no drift or improvements. The regressions cover core extraction logic: column order shifted for coworking and courier receipts, the grand total changed for dinner and parking, VAT amount shifted for office supplies, merchant and date details changed for lunch and train receipts, and a CSV cleaning step now produces the wrong row count. These failures indicate the edit broke existing behavior across multiple receipt types and a data cleaning task. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2cfa611323e6", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1452, "completion_tokens": 147, "usd": 0.0005679 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_f3f61876a85b", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2826, "completion_tokens": 353, "usd": 0.0011655 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_03e985bf60c4", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3811, "completion_tokens": 421, "usd": 0.0015222 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_b14c6d6a1857", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2608, "completion_tokens": 306, "usd": 0.0010578 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_758cddadc695", "task_id": "exp-coworking-courier", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "The rows written to the Expenses sheet have columns in the wrong order compared to the golden run is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3345, "completion_tokens": 328, "usd": 0.0012987 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5e47aca6907b", "task_id": "exp-dinner-parking", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the grand total of 144.00 EUR is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3139, "completion_tokens": 305, "usd": 0.0012162 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a8a18a97157f", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the VAT amount '7.58 EUR' is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2521, "completion_tokens": 293, "usd": 0.00102 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_2a8fe0325d1e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2495, "completion_tokens": 323, "usd": 0.0010392 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_21460568387a", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "super", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3295, "completion_tokens": 383, "usd": 0.0013332 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_1b2e69a62019", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3149, "completion_tokens": 344, "usd": 0.0012543 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_df0530c17b4e", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2282, "completion_tokens": 258, "usd": 0.0009168 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_5b34a19cd981", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2644, "completion_tokens": 296, "usd": 0.0010596 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_b1ea18acb350", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2493, "completion_tokens": 266, "usd": 0.0009873 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_6d6c20462d5a", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2452, "completion_tokens": 289, "usd": 0.0009957 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_1d5047ec8fd2", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "the cleaned file support/tickets.clean.csv contains exactly 3 rows is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2704, "completion_tokens": 279, "usd": 0.0010623 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_a7ca6fd2483c", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2662, "completion_tokens": 297, "usd": 0.0010659 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_8ed25d9902cb", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2205, "completion_tokens": 248, "usd": 0.0008847 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_bdb9bd3422e8", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2508, "completion_tokens": 310, "usd": 0.0010314 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_03c42fb9fb74", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2608, "completion_tokens": 304, "usd": 0.001056 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d54e294c1ad8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1530, "completion_tokens": 193, "usd": 0.0006327 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3cb515f6830e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1652, "completion_tokens": 227, "usd": 0.0006999 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c51d156486d2", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1570, "completion_tokens": 217, "usd": 0.0006663 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0225336, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.075112, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0225336, "ultra": 0.00085 }, "prompt_tokens_by_tier": { "nano": 0, "super": 55951, "ultra": 421 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 20.704, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-1", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_81f243f46bcd", "verdict_id": "verdict_81f243f46bcd", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 2 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 2 of 13 replays. Review before merging. The system ran 13 replay pairs against the updated csv-cleaner skill and found 11 passing, but 2 regressions on steps t003 and m003 where expected row counts are no longer met. The judge determined the cleaned output got worse for these specific cases. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_95b0aeaae315", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2113, "completion_tokens": 278, "usd": 0.0008841 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ad0ef0f6f262", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2362, "completion_tokens": 303, "usd": 0.0009813 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_4bc22f69c2f4", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2307, "completion_tokens": 256, "usd": 0.0009225 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_341f1aafe06d", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "the cleaned file support/tickets.clean.csv contains exactly 3 rows is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2691, "completion_tokens": 304, "usd": 0.0010809 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_89cbeb5f1106", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2571, "completion_tokens": 298, "usd": 0.0010395 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_638b93eff172", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2528, "completion_tokens": 345, "usd": 0.0010689 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_57f1a1c928ec", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2614, "completion_tokens": 263, "usd": 0.0010209 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_ed700d3411da", "task_id": "csv-signups", "outcome": "regression", "expected_outcome": "pass", "decided_by": "super", "reason": "signups.csv contains exactly 4 rows before cleaning is no longer met.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2517, "completion_tokens": 322, "usd": 0.0010449 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_f83b9f46b05b", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2711, "completion_tokens": 309, "usd": 0.0010914 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7323ba2c2619", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2428, "completion_tokens": 250, "usd": 0.0009534 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2e08956d5846", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1500, "completion_tokens": 193, "usd": 0.0006237 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_8ad7f2f013aa", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1503, "completion_tokens": 218, "usd": 0.0006471 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3f0ad4c092bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1516, "completion_tokens": 211, "usd": 0.0006447 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0120033, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.040011, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0120033, "ultra": 0.000576 }, "prompt_tokens_by_tier": { "nano": 0, "super": 29361, "ultra": 300 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.198, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-1", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_1573e0059a55", "verdict_id": "verdict_1573e0059a55", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The edit updated SKILL.md with two additions and zero deletions. The regression suite ran 22 pairs covering the skill's known behaviors and every pair passed with no regressions, drift, or uncertain results. The evaluation cost $0.02. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_9797515450fd", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1418, "completion_tokens": 152, "usd": 0.0005622 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_562f1bacbb4a", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3184, "completion_tokens": 269, "usd": 0.0011973 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_118b690af2c2", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3820, "completion_tokens": 443, "usd": 0.0015447 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_17c39cd8a28a", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2417, "completion_tokens": 304, "usd": 0.0009987 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_89a8beb01ad9", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3125, "completion_tokens": 317, "usd": 0.0012228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5a08a4d190cc", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3118, "completion_tokens": 298, "usd": 0.0012036 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a04984da2f2e", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2413, "completion_tokens": 297, "usd": 0.0009912 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_6c284a0936b6", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2435, "completion_tokens": 267, "usd": 0.0009708 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37dcfb667a2b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3258, "completion_tokens": 364, "usd": 0.001305 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_f63fa53e365f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 3137, "completion_tokens": 343, "usd": 0.0012498 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_596660827ddb", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2298, "completion_tokens": 261, "usd": 0.0009243 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_35dd5c46b4f3", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2528, "completion_tokens": 287, "usd": 0.0010167 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_24bcb89d6737", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2398, "completion_tokens": 260, "usd": 0.0009534 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_3053a2b19841", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2366, "completion_tokens": 312, "usd": 0.0009906 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_df65d3ad0120", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2668, "completion_tokens": 273, "usd": 0.0010461 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_175a9b556a3d", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2662, "completion_tokens": 297, "usd": 0.0010659 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_6396f7029b97", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 2474, "completion_tokens": 340, "usd": 0.0010482 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_7ff7643575cd", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2600, "completion_tokens": 315, "usd": 0.0010635 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_07fbc9fb046d", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2582, "completion_tokens": 297, "usd": 0.0010419 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_11e6e85db7b9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1484, "completion_tokens": 210, "usd": 0.0006342 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_6781c847f496", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1511, "completion_tokens": 219, "usd": 0.0006504 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_16136c6d8859", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1536, "completion_tokens": 211, "usd": 0.0006507 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.022332, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.07444, "spent_usd_by_tier": { "nano": 0.0, "super": 0.022332, "ultra": 0.00043 }, "prompt_tokens_by_tier": { "nano": 0, "super": 55432, "ultra": 220 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 17.099, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-github-triage-1", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_66e8d9d2a760", "verdict_id": "verdict_66e8d9d2a760", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: github-triage still handles all 13 past tasks.", "narrative": "Passed: github-triage still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated SKILL.md and every pair passed with zero regressions, drift, or uncertain results. The change added two lines to the skill file without removing existing logic. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_c6a8e1abb734", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1503, "completion_tokens": 163, "usd": 0.0005976 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_4e23436145b0", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2507, "completion_tokens": 200, "usd": 0.0009321 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e1dd839734fb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3175, "completion_tokens": 258, "usd": 0.0011847 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_860252f0aeb1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3228, "completion_tokens": 336, "usd": 0.0012708 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac0076e8ac87", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2965, "completion_tokens": 302, "usd": 0.0011613 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_0247b5972b80", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3031, "completion_tokens": 328, "usd": 0.0012045 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_07bb9b326000", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3084, "completion_tokens": 358, "usd": 0.0012474 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_ef274337a616", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2992, "completion_tokens": 300, "usd": 0.0011676 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_ee6719208401", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3410, "completion_tokens": 296, "usd": 0.0012894 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_44c02d5b3570", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3147, "completion_tokens": 365, "usd": 0.0012726 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_bfdc8aa6fc39", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1498, "completion_tokens": 198, "usd": 0.0006276 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a63316f74650", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1501, "completion_tokens": 209, "usd": 0.0006384 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_86b52a6672e1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1502, "completion_tokens": 211, "usd": 0.0006405 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0132345, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.044115, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0132345, "ultra": 0.000412 }, "prompt_tokens_by_tier": { "nano": 0, "super": 33543, "ultra": 223 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 7.637, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_888c65848870", "verdict_id": "verdict_888c65848870", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 1 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 1 changed because the world did, not the skill. The system ran 13 evaluation pairs against the updated fx-expense-converter skill, resulting in 12 passes and 1 drift with no regressions. The single drift occurred in the \"Convert London costs from GBP\" task where the GBP/USD exchange rate shifted from 1.2702 to 1.3105 between runs, causing the output to update correctly. The skill logic remains sound; the variance reflects live market data retrieved by Tavily. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_3e8797c16149", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2313, "completion_tokens": 290, "usd": 0.0009549 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_608d63d15c36", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2573, "completion_tokens": 266, "usd": 0.0010113 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_229dc213dbd6", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2406, "completion_tokens": 269, "usd": 0.0009639 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_e71800e8113a", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2505, "completion_tokens": 282, "usd": 0.0010053 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_16ce4eaa1d1a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2450, "completion_tokens": 320, "usd": 0.001023 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_bd178338e4b1", "task_id": "fx-london", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The GBP/USD exchange rate on 2026-10-13 was 1.2702 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 1.0, "prompt_tokens": 2582, "completion_tokens": 311, "usd": 0.0010545 } ], "control_outcome": null, "drift": [ { "fact_id": "t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.2702", "new_value": "1.3105" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_f6108436cd3f", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2592, "completion_tokens": 329, "usd": 0.0010737 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_133f03cc5707", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2387, "completion_tokens": 278, "usd": 0.0009663 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_cb6d6a6fa18e", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2877, "completion_tokens": 286, "usd": 0.0011205 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_9bc309eb22db", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2593, "completion_tokens": 305, "usd": 0.0010524 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_c0d90d4cb42f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1522, "completion_tokens": 193, "usd": 0.0006303 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_7d778bb3b477", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1573, "completion_tokens": 224, "usd": 0.0006735 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9620dc2201ff", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 1562, "completion_tokens": 210, "usd": 0.0006576 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0121872, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.040624, "spent_usd_by_tier": { "nano": 0.00005028, "super": 0.0123837, "ultra": 0.000712 }, "prompt_tokens_by_tier": { "nano": 702, "super": 30539, "ultra": 325 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.17, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_f74b0ee15e6f", "verdict_id": "verdict_f74b0ee15e6f", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 5 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 5 changed because the world did, not the skill. The run covered 8 passing pairs and 5 drift pairs with no regressions or improvements. All five drift cases stem from the EUR/USD exchange rate on 2026-10-14 shifting from 1.0800 to 1.163, a change verified by Tavily. The skill logic correctly applied the new rate in every affected scenario, including conversions for coworking, courier fees, software subscriptions, and travel costs for Hamburg and Berlin. The skill file itself changed only in documentation. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_24416e3f8916", "task_id": "fx-no-date", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2352, "completion_tokens": 363, "usd": 0.0010323 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_c04a94e117fd", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "drift", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2580, "completion_tokens": 254, "usd": 0.0010026 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_823b4ddae43a", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2443, "completion_tokens": 266, "usd": 0.0009723 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_9326a185dca9", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2608, "completion_tokens": 280, "usd": 0.0010344 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_48d13ccbd36d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2418, "completion_tokens": 313, "usd": 0.0010071 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_24d29ab3083c", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 1.0, "prompt_tokens": 2573, "completion_tokens": 272, "usd": 0.0010167 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_d540794f1c98", "task_id": "fx-coworking-courier", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2581, "completion_tokens": 387, "usd": 0.0011226 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ecc9e77802a1", "task_id": "fx-software", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "finance.rate returned 1.08 on 2026-10-14 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2396, "completion_tokens": 328, "usd": 0.001014 } ], "control_outcome": null, "drift": [ { "fact_id": "obs_t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.08", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_ce1c4a74b4df", "task_id": "fx-hamburg-conference", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2908, "completion_tokens": 386, "usd": 0.0012198 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_84352baa167a", "task_id": "fx-berlin-trip", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 1.0, "prompt_tokens": 2564, "completion_tokens": 315, "usd": 0.0010527 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_68a265f4d0b3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "drift", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1522, "completion_tokens": 214, "usd": 0.0006492 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_eac0515b4a0c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1573, "completion_tokens": 221, "usd": 0.0006708 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_aa0845d7fbd7", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1562, "completion_tokens": 217, "usd": 0.0006639 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0124584, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.041528, "spent_usd_by_tier": { "nano": 0.00052548, "super": 0.0134058, "ultra": 0.001044 }, "prompt_tokens_by_tier": { "nano": 5694, "super": 32983, "ultra": 627 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.223, "simulated_models": false, "repeat": 0, "error": null } ] }, "ultra_only": { "config": { "name": "ultra_only", "description": "Ultra only — every pair judged by ultra alone (Level 0 and drift as in the cascade).", "experiments": [ "E4" ], "mode": "edits", "cascade": { "mode": "ultra_only", "level0": true, "drift": true, "controls": true }, "single_judge_blocks": true, "types": [ "R1", "R2", "R3", "R4", "R5", "R6", "N0", "D1" ], "noise_repeats": 3, "runner": "local", "concurrency": 2, "reuse_replays": "cascade", "max_traces": 40 }, "config_path": "bench/configs/ultra_only.yaml", "command": "ratchet bench run --config bench/configs/ultra_only.yaml --out bench/results/2026-10-16 --types D1 --limit 2 --yes", "started_at": "2026-10-16T12:55:50.690132Z", "updated_at": "2026-10-16T13:58:02.852549Z", "edits": [ { "bench_id": "n0-github-triage-2", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_a7f43edd0f1a", "verdict_id": "verdict_a7f43edd0f1a", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 12 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 12 of 13 replays. Review before merging. The system re-ran 13 test cases against the proposed skill edit, which only changed \"when\" to \"whenever\" and renamed \"Steps\" to \"Procedure\" in the trigger and procedure sections. One case passed, ten were flagged as regressions, and two were marked uncertain. The judge marked cases as regressed even when the replay output was identical to the golden output and met all success criteria, suggesting the evaluation logic itself may be misaligned with the minor wording changes. Two cases were held for human review due to ambiguous signals. The semantic shift in a separate rewrite task confirms the judge is sensitive to minor phrasing differences. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_811b35141f2c", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff changed the trigger wording from 'when' to 'whenever' and renamed 'Steps' to 'Procedure', which may have altered the model's interpretation of when to invoke the skill is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1413, "completion_tokens": 1133, "usd": 0.004812 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_b3d50ce1b799", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2446, "completion_tokens": 657, "usd": 0.004417 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_7741c40ef280", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Both runs produce identical final outputs meeting all success criteria: repository name stated, 4 open issues counted, #500 labeled bug, #502 and #504 labeled question, #503 noted as already labeled. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3115, "completion_tokens": 739, "usd": 0.005332 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_abe3d0283a76", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3106, "completion_tokens": 618, "usd": 0.00496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_a0217d5141b7", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "All success criteria are met identically in both runs: 4 open issues stated, #88 labeled bug, #90 labeled enhancement, #91 labeled question, #92 noted as already labeled, and details requested for #91 via comment is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.05, "prompt_tokens": 2776, "completion_tokens": 962, "usd": 0.005662 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_f5d663dbc019", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only rewords 'when' to 'whenever' and 'Steps' to 'Procedure' — no procedural change that would alter tool-calling behavior. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.6, "prompt_tokens": 3166, "completion_tokens": 3195, "usd": 0.012751 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7e7f0890e15f", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay final output is identical to the golden final output and meets all six success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2834, "completion_tokens": 860, "usd": 0.005414 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a1def1b764dc", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay meets all success criteria identically to the golden run: both list 4 open issues, label #58 as bug, #61 as enhancement, #64 as question, note #62 as already labeled, and ask for details on #64 via github.comment is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2891, "completion_tokens": 824, "usd": 0.005363 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_0490537a6338", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3245, "completion_tokens": 700, "usd": 0.005345 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_8b534e552950", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output omits the opening line 'Here's the triage summary:' present in the golden output, but this is a cosmetic formatting difference that does not affect any success criterion is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2984, "completion_tokens": 758, "usd": 0.005258 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_7e792837459f", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1398, "completion_tokens": 477, "usd": 0.002829 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_95212662e369", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1408, "completion_tokens": 2062, "usd": 0.007594 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a24da7604ca9", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output changes 'the agenda is unchanged' to 'the agenda will stay unchanged', which alters the tense and adds 'will stay', a semantic shift not present in the golden output is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1431, "completion_tokens": 645, "usd": 0.003366 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.073103 }, "counterfactual_ultra_usd": 0.073103, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.074412 }, "prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 33033 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 30.8, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-1", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_27273ccfae27", "verdict_id": "verdict_27273ccfae27", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 11 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 11 of 13 replays. Review before merging. The system replayed 13 test cases against the proposed change to github-triage, which only removes the \"Asked for details\" output line from SKILL.md. Two cases passed, but ten regressed because the agent stopped producing the \"Asked for details\" lines that the golden outputs and success criteria still require. One case is uncertain because the diff does not touch labeling rules, yet the output changed. The evidence shows the skill specification was altered to drop a required output format, causing widespread test failures. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_29198b3e615a", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly (step m001) is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1349, "completion_tokens": 772, "usd": 0.003665 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_ae5049772f22", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output is missing the 'Asked for details: none' line that appears in the golden output is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2238, "completion_tokens": 1843, "usd": 0.007767 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_6a4a7b2c3a20", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only removes the 'Asked for details' output format line; it does not modify the triage labeling rules. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2994, "completion_tokens": 2677, "usd": 0.011025 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ff7547adf9e1", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits 'Asked for details: #77' which the golden output includes is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3003, "completion_tokens": 2039, "usd": 0.00912 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_95dc2c9a2266", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay final output omits the 'Asked for details: #91' line that the success criteria requires and the golden output includes is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2769, "completion_tokens": 1439, "usd": 0.007086 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_1ac70e25ffb3", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2971, "completion_tokens": 2722, "usd": 0.011137 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_3be8109cfb2f", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output at step m004 omits 'Asked for details: none' which is required by the success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2945, "completion_tokens": 1259, "usd": 0.006722 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_650e3f7e3ede", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits the 'Asked for details: #64' line that the success criteria requires is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2839, "completion_tokens": 1546, "usd": 0.007477 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_e5bee22bc04e", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits 'Asked for details: #36' which is required by success criterion 4 is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3250, "completion_tokens": 1095, "usd": 0.006535 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_0c922bfa825f", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits 'Asked for details: #15' which is required by success criterion 4 is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2945, "completion_tokens": 1116, "usd": 0.006293 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_00c8cee98bc3", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 512, "usd": 0.002852 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_05517158aa56", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1362, "completion_tokens": 1362, "usd": 0.005448 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_bdbe9648c85c", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff removes the requirement to include 'Asked for details' in the output, which is a change to the skill's specification is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1419, "completion_tokens": 1736, "usd": 0.006627 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.091754 }, "counterfactual_ultra_usd": 0.091754, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.092831 }, "prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 32069 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 43.853, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-2", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_fcd8b58c5d6c", "verdict_id": "verdict_fcd8b58c5d6c", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 19 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 19 of 22 replays. Review before merging. The system replayed 22 test cases against the proposed change to expense-report SKILL.md, which alters VAT reading priority from top-level field first to summary.vat first. Three cases passed. Fourteen regressed: five expense filing tasks failed because the VAT logic change broke receipts relying on the top-level vat field, and nine CSV cleaning tasks regressed because the judge incorrectly flagged the unrelated skill diff as the cause despite identical outputs. Five cases were held uncertain where outputs matched perfectly but the judge deferred to human review. The evidence shows the VAT logic change breaks specific receipt layouts (conference, office supplies) and the judge is hallucinating regressions on unrelated csv-cleaner tasks. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_db2fa52f33ec", "task_id": "exp-no-files", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay matches the golden run exactly in the first step, but the skill diff changes VAT reading logic which could affect later processing steps not shown in the traces is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1439, "completion_tokens": 1384, "usd": 0.005591 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_663c04fc7679", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output is missing a blank line before 'Not filed:' compared to the golden output, but formatting differences like blank lines are explicitly not regressions per the rules is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3223, "completion_tokens": 1255, "usd": 0.006988 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_b8eacf7ca9eb", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3855, "completion_tokens": 1331, "usd": 0.007848 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_df4a9a6d2d5e", "task_id": "exp-software", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "All success criteria are met identically in both runs: merchant, date, total, VAT, grand total, and filing confirmation match exactly. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2461, "completion_tokens": 854, "usd": 0.005023 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_2aba1c6b9a48", "task_id": "exp-coworking-courier", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Both runs produce identical outputs meeting all success criteria: correct merchants, dates, totals, and VAT amounts for both receipts. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3157, "completion_tokens": 749, "usd": 0.005404 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_4d8a32096183", "task_id": "exp-dinner-parking", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run produces identical tool outputs and final output to the golden run, meeting all success criteria. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3164, "completion_tokens": 860, "usd": 0.005744 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_5d79f77bf29a", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff changed VAT reading logic from 'top-level vat field first, then summary.vat fallback' to 'summary.vat first, then top-level vat fallback' is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2459, "completion_tokens": 666, "usd": 0.004457 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_fa5cfd551a14", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff changed VAT reading priority from top-level vat field first to summary.vat first, but the receipt parsed has layout A with a populated top-level vat field (79.83) and no summary.vat shown in the tool output is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2426, "completion_tokens": 724, "usd": 0.004598 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_bcd2f6c4de1b", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3337, "completion_tokens": 929, "usd": 0.006124 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6c9bfaed6179", "task_id": "exp-berlin-trip", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3118, "completion_tokens": 844, "usd": 0.00565 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_979bc36816b3", "task_id": "csv-vendors", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output: both produce vendors.clean.csv with columns id,vendor,email,date and two data rows is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2337, "completion_tokens": 1122, "usd": 0.005703 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_0444b760bf07", "task_id": "csv-inventory", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (VAT reading logic) but the task uses csv-cleaner skill on inventory.csv; the diff is unrelated to the executed skill is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2588, "completion_tokens": 948, "usd": 0.005432 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_997e188d6e47", "task_id": "csv-customers", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (VAT reading logic), but the task uses csv-cleaner skill to clean customers.csv — a different skill entirely is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2460, "completion_tokens": 946, "usd": 0.005298 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_d40c1503ab73", "task_id": "csv-already-clean", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (VAT reading logic) but the task uses csv-cleaner skill to clean contacts.csv — unrelated skills. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2450, "completion_tokens": 1291, "usd": 0.006323 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_8111d388d469", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (VAT reading logic) but the task is to clean support/tickets.csv using a CSV cleaner skill - the diff is unrelated to this task is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2633, "completion_tokens": 839, "usd": 0.00515 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0727abbc4d2a", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (VAT reading logic) but the task uses a CSV cleaner skill; the diff is unrelated to the payments.csv cleaning task is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2633, "completion_tokens": 1487, "usd": 0.007094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_cfff2f4aa409", "task_id": "csv-header-only", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay produces identical output to the golden run: both read a leads.csv containing only a header row, both write a leads.clean.csv with only the header, and both report 0 rows in/out. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2145, "completion_tokens": 1989, "usd": 0.008112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_52606712bd67", "task_id": "csv-attendees", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (VAT reading logic) but the task uses a csv-cleaner skill for attendees.csv; the diff is unrelated to the executed skill is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2431, "completion_tokens": 964, "usd": 0.005323 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_542487f91fa5", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2475, "completion_tokens": 2229, "usd": 0.009162 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_599436fdbbc6", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly, meeting all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1487, "completion_tokens": 444, "usd": 0.002819 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_de6195626ea9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 1515, "completion_tokens": 856, "usd": 0.004083 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e212cacce84", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and satisfies all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1590, "completion_tokens": 721, "usd": 0.003753 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.125679 }, "counterfactual_ultra_usd": 0.125679, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.127299 }, "prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 56481 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 52.41, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_d0c924e4262a", "verdict_id": "verdict_d0c924e4262a", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 10 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 10 of 13 replays. Review before merging. The system replayed 13 test cases against the skill change that renamed the target sheet from 'Expenses USD' to 'Expenses'. One case passed, one improved, eight regressed, two were uncertain, and one drifted due to live exchange rate updates. The strongest signal is the sheet rename: multiple replays now write to 'Expenses' where the golden runs expected 'Expenses USD', causing append failures or criterion mismatches. Two cases were held for human review: one shows a date mismatch in the finance rate response, the other is an unrelated summarization task affected by the diff. The improved case meets all criteria with correct conversion values. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_885b30526626", "task_id": "fx-no-date", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff changed the target sheet from 'Expenses USD' to 'Expenses', causing the sheets.append call to target a different sheet (t002) is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2318, "completion_tokens": 1124, "usd": 0.00569 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_fcf301073639", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay shows the rate date as 2026-10-09 (from finance.rate response) instead of the requested 2026-10-11, failing the 'correct date' criterion. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2640, "completion_tokens": 2544, "usd": 0.010272 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_82a70ccb2af8", "task_id": "fx-tokyo", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output meets all six success criteria: it states 18400 JPY = 123.32 USD, includes the rate 0.0067, specifies the date 2026-10-14, and uses the exact required amounts is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2338, "completion_tokens": 1218, "usd": 0.005992 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "0.0067", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_973ca2670ff2", "task_id": "fx-paris", "outcome": "drift", "expected_outcome": "regression", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-09 was 1.0768 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2617, "completion_tokens": 836, "usd": 0.005125 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3afca948a379", "task_id": "fx-zurich", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay run writes to sheet 'Expenses' (per the skill diff) but the sheets.append tool returns appended: 0, indicating the write failed or the sheet doesn't exist, whereas the golden run wrote to 'Expenses USD' and returned appended: 1 is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2395, "completion_tokens": 1016, "usd": 0.005443 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.15714", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_f5be008e5f9d", "task_id": "fx-london", "outcome": "improved", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Meets the criteria and improves All five success criteria are met by the replay output: hotel/train/total amounts with correct USD values, exchange rate with date, and correct conversion values (279.44, 61.60, 341.04) are all present in step m003.", "opinions": [ { "tier": "ultra", "verdict": "improved", "confidence": 0.95, "prompt_tokens": 2586, "completion_tokens": 1147, "usd": 0.006027 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_b55b161ede07", "task_id": "fx-coworking-courier", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff changed the sheet name from 'Expenses USD' to 'Expenses' in step 4, and the replay correctly calls sheets.append with 'Expenses' while the golden run used 'Expenses USD' is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2601, "completion_tokens": 1179, "usd": 0.006138 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_e70094e8923d", "task_id": "fx-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff changed the target sheet from 'Expenses USD' to 'Expenses' is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2404, "completion_tokens": 1591, "usd": 0.007177 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5bde00727c59", "task_id": "fx-hamburg-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff changed the sheet name from 'Expenses USD' to 'Expenses' in the sheets.append call instruction is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2899, "completion_tokens": 1040, "usd": 0.006019 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_96be56214418", "task_id": "fx-berlin-trip", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output matches all success criteria: it states taxi (50.00 EUR = 54.00 USD), hotel (350.00 EUR = 378.00 USD), total (400.00 EUR = 432.00 USD), and the rate (1 EUR = 1.0800 USD on 2026-10-14) at m003 is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2616, "completion_tokens": 941, "usd": 0.005439 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d1a68d58b4ae", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output uses 'shipped' instead of 'deploy', but both convey inability to deploy the update, meeting criterion 1 is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1518, "completion_tokens": 745, "usd": 0.003753 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_96c18d79778a", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1564, "completion_tokens": 1306, "usd": 0.005482 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9cb3c3858cd1", "task_id": "canary-summary", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies fx-expense-converter/SKILL.md (changing a sheet name from 'Expenses USD' to 'Expenses'), but the task is summarizing a note about an offsite meeting;. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1531, "completion_tokens": 1082, "usd": 0.004777 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.077334 }, "counterfactual_ultra_usd": 0.077334, "spent_usd_by_tier": { "nano": 0.00040368, "super": 0.0017751, "ultra": 0.078876 }, "prompt_tokens_by_tier": { "nano": 5660, "super": 5443, "ultra": 31074 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 36.138, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-1", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_17219139619a", "verdict_id": "verdict_17219139619a", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would close issues in 5 of 13 replays.", "narrative": "Blocked: the new github-triage would close issues in 5 of 13 replays. The edit added a rule to close issues labeled \"question\" after commenting. Regression tests caught 11 failures: five replays made new close_issue calls the sandbox rejected, two replays failed to close required issues, two triage replays no longer matched golden outputs, and two unrelated writing tasks regressed. Only two pairs passed. The change introduces incorrect closing behavior across multiple repositories and breaks unrelated skills. Roll back the edit and rework the triage logic to handle \"question\" labels without over-closing or under-closing. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_f8c354ae33e9", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff added a rule to close issues labeled 'question' after commenting, which changes the triage behavior by closing issues that should remain open is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1450, "completion_tokens": 2344, "usd": 0.008482 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_478d9ee2ce19", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2478, "completion_tokens": 648, "usd": 0.004422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_755deee2dac3", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ce4b9acb3157", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 3 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 3, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_11ee7c7f4f30", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_9432a20fd19d", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.75, "prompt_tokens": 3209, "completion_tokens": 1955, "usd": 0.009074 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_715dfb3b7eec", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_5ba0f0c4e92d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_39066d273f98", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff added a rule to close issues labeled 'question' after posting the comment, but the replay run did not close issue #36 (step m004 explicitly states 'I won't close any'), missing a required side effect introduced by the diff is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.85, "prompt_tokens": 6818, "completion_tokens": 8557, "usd": 0.032489000000000004 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_c944d9078bcc", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff added a rule to close issues labeled 'question' after posting the comment, but the replay run did not close issue #15 is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3136, "completion_tokens": 2412, "usd": 0.010372 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_56c2b63686fb", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output uses 'shipped' instead of 'deploy', but the success criteria accept synonyms (ship/deploy) as meeting the meaning criterion, so this criterion is met is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1435, "completion_tokens": 1064, "usd": 0.004627 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a8e17ca9255f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1463, "completion_tokens": 808, "usd": 0.003887 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_dd5897c02420", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1466, "completion_tokens": 759, "usd": 0.003743 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.077096 }, "counterfactual_ultra_usd": 0.077096, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.078259 }, "prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 22213 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 53.338, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-1", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_e6559024b733", "verdict_id": "verdict_e6559024b733", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The edit only changed the skill description in SKILL.md, yet 10 of 13 test cases regressed, including unrelated tasks like summarization and math. One case improved by correctly requesting details on an issue. Two cases are uncertain because the replay added extra labels not in the golden output. The sandbox blocked a new github.comment call in the docs-site replay, which caused that failure. The widespread regressions across unrelated skills suggest a systemic evaluation or environment issue rather than a logic defect in the description change. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_165175858316", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output, so all success criteria are met is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1346, "completion_tokens": 1430, "usd": 0.005636 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_187d5cdcf6dc", "task_id": "tri-status-page", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output includes two extra lines ('#8: bug' and '#11: enhancement') not present in the golden output, violating the 'no additional text' criterion. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2438, "completion_tokens": 1293, "usd": 0.006317 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3f0a0fcb0075", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output shows '#503: Already labeled: #503' while the golden output shows 'Already labeled: #503' - this is a wording difference but conveys the same meaning is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3039, "completion_tokens": 1045, "usd": 0.006174 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_9972d203f927", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only changed the description field in SKILL.md, adding broader scope text but not altering any triage logic, rules, or tool usage is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3076, "completion_tokens": 767, "usd": 0.005377 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c71e5c2370b8", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "All success criteria are met in the replay run: it states 4 open issues, labels #88 as bug, #90 as enhancement, #91 as question, notes #92 as already labeled, and asks for details on #91 is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2838, "completion_tokens": 858, "usd": 0.005412 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_eb9d9e6851a5", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets criteria 1-5: it states 4 open issues, lists each with type, notes #406 already labeled, states no details requested, and shows github.label artifacts changed. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2917, "completion_tokens": 2557, "usd": 0.010588 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_de7759ea9e17", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_092b5ebe7778", "task_id": "tri-mobile-app", "outcome": "improved", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Meets the criteria and improves All six success criteria are met in the replay run: it states 4 open issues, labels #58 as bug, #61 as enhancement, #64 as question, notes #62 as already labeled, and asks for details on #64 with a github.comment artifact.", "opinions": [ { "tier": "ultra", "verdict": "improved", "confidence": 0.95, "prompt_tokens": 2918, "completion_tokens": 1074, "usd": 0.00614 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_968755dee57c", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3465, "completion_tokens": 806, "usd": 0.005883 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_10dca42d0cf4", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only changed the description text, adding 'or handle any GitHub-related task such as reviewing pull requests, managing labels, or organizing project boards.' This is a cosmetic documentation change that does not alter any too… is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3174, "completion_tokens": 1011, "usd": 0.006207 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_990c76131fc0", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1367, "completion_tokens": 542, "usd": 0.002993 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_47c34378e66c", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly and satisfies all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1359, "completion_tokens": 1001, "usd": 0.004362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_602f5ee8bbac", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies the github-triage skill description, which is unrelated to the summarization task being evaluated is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 946, "usd": 0.004254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.069343 }, "counterfactual_ultra_usd": 0.069343, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.07061 }, "prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 30227 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 25.117, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_7a2f23cef743", "verdict_id": "verdict_7a2f23cef743", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 10 of 14 replays. Review before merging.", "narrative": "Held: one judge found a regression on 10 of 14 replays. Review before merging. The system replayed 14 test cases against the skill change, which only added the word \"expense\" to the description text. Two cases passed, one improved, eight regressed, one drifted due to live exchange rate updates, and two were uncertain. The regressions include identical outputs suddenly failing criteria, precision differences in JPY rates, capitalization mismatches flagged despite rules ignoring them, and unrelated tasks like summarization failing because of the description edit. The weekend date case and a rewrite task were held for human review due to date discrepancies and extra output content. The evidence suggests the judge evaluation is unreliable or the criteria are brittle. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_9b9fe0986841", "task_id": "fx-no-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay meets all success criteria with identical output to the golden run is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2144, "completion_tokens": 1150, "usd": 0.005594 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_f97cae02fbc4", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output states the rate date as 2026-10-09 while the golden output states 2026-10-11, a changed date value in the final message. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2401, "completion_tokens": 2535, "usd": 0.010006 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6eb01a2d41d3", "task_id": "fx-tokyo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output shows the exchange rate as 0.006702 while the golden output shows 0.0067, a difference in precision that changes the displayed value is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2239, "completion_tokens": 817, "usd": 0.00469 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "0.0067", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_315104ec0683", "task_id": "fx-paris", "outcome": "drift", "expected_outcome": "pass", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-09 was 1.0768 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2334, "completion_tokens": 1057, "usd": 0.005505 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_762ca612dbe8", "task_id": "fx-zurich", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only changed the description text by adding 'expense' before 'amounts', which is a cosmetic wording change that does not affect functionality is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2241, "completion_tokens": 780, "usd": 0.004581 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.15714", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_4ef2ca8f8b72", "task_id": "fx-london", "outcome": "improved", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Meets the criteria and improves All success criteria are met in the replay: hotel amount (220.00 GBP = 279.44 USD), train amount (48.50 GBP = 61.60 USD), total (268.50 GBP = 341.04 USD), exchange rate with date (1.2702 on 2026-10-13), and correct conversion values are all….", "opinions": [ { "tier": "ultra", "verdict": "improved", "confidence": 1.0, "prompt_tokens": 2406, "completion_tokens": 876, "usd": 0.005034 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_53e1872e54f8", "task_id": "fx-coworking-courier", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output uses lowercase 'coworking pass' and 'courier' while the golden output uses title case 'Coworking pass' and 'Courier', but capitalization differences are explicitly not regressions per the rules is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2402, "completion_tokens": 986, "usd": 0.00536 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_fddd61f372fe", "task_id": "fx-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets all success criteria: it states the EUR amount (117.81), USD amount (127.23), exchange rate (1.0800), date (2026-10-14), includes 'software subscription', and includes 'Total: 117.81 EUR = 127.23 USD' (m003) is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2195, "completion_tokens": 856, "usd": 0.004763 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_bda2112c7543", "task_id": "fx-hamburg-conference", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds the word 'expense' to the description text, which does not change any tool calls, arguments, or logic is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2708, "completion_tokens": 785, "usd": 0.005063 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c3051db65b47", "task_id": "fx-berlin-trip", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay final output is identical to the golden final output, meeting all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2408, "completion_tokens": 1174, "usd": 0.00593 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_8981ccff882f", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3686, "completion_tokens": 1162, "usd": 0.007172 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_ec291cce2b97", "task_id": "canary-formal-rewrite", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output includes a greeting and additional explanation, violating the requirement to have no extra content beyond the formal statement. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 1348, "completion_tokens": 1523, "usd": 0.005917 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_cbb67ab2491c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1332, "completion_tokens": 1501, "usd": 0.005835 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_91fcb24262ed", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff adds the word 'expense' to the description of fx-expense-converter, which is unrelated to the summarization task is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1371, "completion_tokens": 662, "usd": 0.003357 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.078807 }, "counterfactual_ultra_usd": 0.078807, "spent_usd_by_tier": { "nano": 0.0004032, "super": 0.001776, "ultra": 0.080278 }, "prompt_tokens_by_tier": { "nano": 5660, "super": 5446, "ultra": 32200 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 36.434, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_5e90fab772e5", "verdict_id": "verdict_5e90fab772e5", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 11 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 11 of 13 replays. Review before merging. The system replayed 13 test cases against the edited skill; zero passed, 11 regressed, and two drifted due to live EUR/USD rate changes. Most regressions show the replay output is identical to the golden output yet the judge marked the success criterion as no longer met, suggesting the evaluation logic or criteria shifted. Two cases drifted because Tavily retrieved updated exchange rates (1.0768 to 1.0800). One case failed on missing bold markdown formatting for a calculation. One case failed on a missing tool call despite correct output. One case failed on title-case versus lowercase category labels. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_382f2e4d44ae", "task_id": "fx-no-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2221, "completion_tokens": 764, "usd": 0.004513 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_75e2cc6a76a8", "task_id": "fx-weekend-date", "outcome": "drift", "expected_outcome": "pass", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-11 was 1.0768 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2438, "completion_tokens": 1140, "usd": 0.005858 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_c536f0cd0389", "task_id": "fx-tokyo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2283, "completion_tokens": 781, "usd": 0.004626 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "0.0067", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_a8c493d9890b", "task_id": "fx-paris", "outcome": "drift", "expected_outcome": "pass", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-09 was 1.0768 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2453, "completion_tokens": 865, "usd": 0.005048 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_b7a6a78b225c", "task_id": "fx-zurich", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2318, "completion_tokens": 762, "usd": 0.004604 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.15714", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a00a986b53f7", "task_id": "fx-london", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2420, "completion_tokens": 915, "usd": 0.005165 } ], "control_outcome": null, "drift": [ { "fact_id": "t001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.2702", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_c7bcfb7bf68e", "task_id": "fx-coworking-courier", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output uses lowercase 'coworking pass' and 'courier' while the golden output uses title case 'Coworking pass' and 'Courier' is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2495, "completion_tokens": 865, "usd": 0.00509 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_3b832d8b5c25", "task_id": "fx-software", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2308, "completion_tokens": 791, "usd": 0.004681 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_f76c8a253ef0", "task_id": "fx-hamburg-conference", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run omitted the initial time.now tool call that the golden run made, but still produced identical correct output with the same exchange rate and date is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2695, "completion_tokens": 1313, "usd": 0.006634 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c432f688c91f", "task_id": "fx-berlin-trip", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2503, "completion_tokens": 878, "usd": 0.005137 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1f1fa89dddfb", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly (step m001) is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 1226, "usd": 0.005077 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_9470f1a93d4f", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output lacks the bold markdown formatting around 40.8 that the success criterion explicitly requires is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1445, "completion_tokens": 870, "usd": 0.004055 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f86b4f70f4f6", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly, satisfying all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1412, "completion_tokens": 822, "usd": 0.003878 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.064366 }, "counterfactual_ultra_usd": 0.064366, "spent_usd_by_tier": { "nano": 0.00045432, "super": 0.0020043, "ultra": 0.065629 }, "prompt_tokens_by_tier": { "nano": 6364, "super": 6138, "ultra": 29170 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 36.236, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-1", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_cde8598acc71", "verdict_id": "verdict_cde8598acc71", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 18 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 18 of 22 replays. Review before merging. The judge compared 22 test pairs and found 15 regressions, 2 improvements, 3 uncertain, and only 2 passes. The strongest pattern shows the skill diff reordered VAT and Total output lines in expense-report, causing 8 replays to follow the new order while golden outputs follow the old order. Three CSV cleaning tasks regressed despite the diff only touching expense-report, suggesting the judge misattributed the cause. Two cases improved because the replay matched the new intended format. Two cases were held for human review due to incomplete traces or unrelated diffs. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_ddb84b0fe47b", "task_id": "exp-no-files", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The trace only contains the first step where the model asks for receipt filenames;. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.8, "prompt_tokens": 1446, "completion_tokens": 3390, "usd": 0.011616 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_7d67f2353f20", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits the 'Not filed' line for the second receipt entirely, whereas the golden output includes it and the success criteria require stating that the second receipt was not found is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3003, "completion_tokens": 1455, "usd": 0.007368 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_acc26084d722", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3845, "completion_tokens": 1772, "usd": 0.009161 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_af5ade88f377", "task_id": "exp-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff explicitly reordered the output instructions to place VAT before Total, and the replay output follows this new order while the golden output follows the old order is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2362, "completion_tokens": 1147, "usd": 0.005803 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_a69ed051d7f4", "task_id": "exp-coworking-courier", "outcome": "improved", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Meets the criteria and improves The skill diff explicitly swapped the order of VAT and Total lines in the output format specification (step m001 in golden shows old order, replay follows new order).", "opinions": [ { "tier": "ultra", "verdict": "improved", "confidence": 1.0, "prompt_tokens": 3101, "completion_tokens": 959, "usd": 0.005978 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_3e7d7797755d", "task_id": "exp-dinner-parking", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff moved the VAT line before the Total line in the output format (step 5), and the replay output follows this new order while the golden output follows the old order is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3052, "completion_tokens": 1249, "usd": 0.006799 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_079c36312676", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff explicitly reorders the VAT and Total lines in the output format, moving VAT before Total is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2353, "completion_tokens": 1314, "usd": 0.006295 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_f07720304e5f", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff moved the VAT line before the Total line in the output format instructions, and the replay output follows this new order (VAT then Total) while the golden output follows the old order (Total then VAT) is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2387, "completion_tokens": 1257, "usd": 0.006158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_8626f6fe5abb", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff explicitly reordered the required output lines so that VAT appears before Total, and the replay output follows this new order while the golden output follows the old order is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3216, "completion_tokens": 1858, "usd": 0.00879 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_521e59e6091f", "task_id": "exp-berlin-trip", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff explicitly moved the VAT line before the Total line in the output format instructions is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3121, "completion_tokens": 1375, "usd": 0.007246 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_e31167be7710", "task_id": "csv-vendors", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output: both produce vendors.clean.csv with 2 data rows, header, no duplicate ids, no missing ids, id column, and vendor column (not name) is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2184, "completion_tokens": 1208, "usd": 0.005808 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_229e9b0e57cc", "task_id": "csv-inventory", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Both golden and replay runs produce identical output meeting all success criteria. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2545, "completion_tokens": 567, "usd": 0.004246 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_1aec4a330ef6", "task_id": "csv-customers", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md but the task uses csv-cleaner skill, so the diff cannot explain any behavioral difference is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2350, "completion_tokens": 828, "usd": 0.004834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_5b8721009163", "task_id": "csv-already-clean", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md but the task uses csv-cleaner skill, so the diff is unrelated to this task is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2318, "completion_tokens": 1046, "usd": 0.005456 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3aee4f6f9c1d", "task_id": "csv-tickets", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run reports 'Rows out: 4' while the golden run reports 'Rows out: 3', but both write identical output files with 3 data rows plus header is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2556, "completion_tokens": 3110, "usd": 0.011886 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_838608cb3f0e", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay reports 'Rows in: 4' while the golden run reports 'Rows in: 5' for the same input file with 5 data rows is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2536, "completion_tokens": 1275, "usd": 0.006361 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_010f6ae26a1d", "task_id": "csv-header-only", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run produces identical output to the golden run: both create leads.clean.csv with only the header row and report 0 rows in/out is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2105, "completion_tokens": 1211, "usd": 0.005738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_a8ba615a5d23", "task_id": "csv-attendees", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (VAT line ordering) which is unrelated to the CSV cleaning task. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2487, "completion_tokens": 928, "usd": 0.005271 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_22e0e0ff85d9", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.85, "prompt_tokens": 2563, "completion_tokens": 1657, "usd": 0.007534 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_15046a030360", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly, satisfying all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1410, "completion_tokens": 717, "usd": 0.003561 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3ac334e2f481", "task_id": "canary-percent", "outcome": "improved", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Meets the criteria and improves The replay output '17% of 240 = 0.17 × 240 = 40.8.' contains the same numeric result (40.8) and all required phrases except bold formatting around 40.8.", "opinions": [ { "tier": "ultra", "verdict": "improved", "confidence": 0.95, "prompt_tokens": 1474, "completion_tokens": 1140, "usd": 0.004894 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb845c98bd90", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1495, "completion_tokens": 989, "usd": 0.004462 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.145265 }, "counterfactual_ultra_usd": 0.145265, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.14685 }, "prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 55047 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 59.363, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-expense-report-1", "skill": "expense-report", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_84de1a08db4f", "verdict_id": "verdict_84de1a08db4f", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 19 of 22 replays. Review before merging.", "narrative": "Held: one judge found a regression on 19 of 22 replays. Review before merging. The system replayed 22 test pairs against the edited expense-report skill, finding 3 pass, 14 regressions, and 5 uncertain. The skill diff removed the requirement to output separate VAT lines, changing the format to a combined total line. This broke 14 replays that explicitly required distinct VAT amounts for receipts and CSV cleaning tasks. Five replays were held for human review due to wording differences or date normalization discrepancies. One replay failed because the diff modified expense-report rules but the task used the csv-cleaner skill. Cost was $0.15. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_73fd927ba82e", "task_id": "exp-no-files", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay's initial response differs only in wording, not in substance; both ask for receipt filenames. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 1458, "completion_tokens": 2728, "usd": 0.009642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_71599b6cf2f5", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff removed the requirement to report VAT on its own line, changing the output format to 'Total: (includes VAT)' without a separate VAT line is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3189, "completion_tokens": 1612, "usd": 0.008025 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_68029dbf8d42", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3694, "completion_tokens": 1955, "usd": 0.009559 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_0061479721a1", "task_id": "exp-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff removed the requirement to output a separate VAT line, changing the format to 'Total: (includes VAT)' and deleting the 'VAT: ' line is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2308, "completion_tokens": 752, "usd": 0.004564 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_cde84ba8b5d5", "task_id": "exp-coworking-courier", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "All six success criteria are met by the replay output: both merchant names, both dates, and both total amounts appear correctly. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3011, "completion_tokens": 1706, "usd": 0.008129 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_d2bfe3e90095", "task_id": "exp-dinner-parking", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits the separate VAT lines for each receipt, showing only 'Total: X EUR (includes VAT)' instead of separate 'Total' and 'VAT' lines is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3070, "completion_tokens": 1016, "usd": 0.006118 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_ea8d1c2b4f4b", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits the required 'VAT: 7.58 EUR' line that the success criteria demands is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2349, "completion_tokens": 1006, "usd": 0.005367 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_bcff8a71d40d", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits the required VAT line entirely, while the success criteria explicitly requires 'states the VAT as 79.83 EUR' is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2398, "completion_tokens": 828, "usd": 0.004882 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_49a1a41ef5ee", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff changed the required output format from separate VAT lines to 'Total: (includes VAT)' without separate VAT amounts is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3184, "completion_tokens": 1821, "usd": 0.008647 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_87ffd7a9b61f", "task_id": "exp-berlin-trip", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output omits the separate VAT lines (includes the separate VAT amount lines for each receipt, which the success criteria requires is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3043, "completion_tokens": 1397, "usd": 0.007234 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_bfab509e3676", "task_id": "csv-vendors", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output lacks a column named 'name' as required by the success criteria; the header shows 'vendor' instead is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2276, "completion_tokens": 1188, "usd": 0.00584 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_ae36c7d648f7", "task_id": "csv-inventory", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay produces a different date value for row A3: 2026-01-10 (replay t003) vs 2026-10-01 (golden t003), indicating a date normalization regression. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2571, "completion_tokens": 5125, "usd": 0.017946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_c94efe16dd1d", "task_id": "csv-customers", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run produces a different date for row 1 (2026-10-03 vs 2026-03-10) while all success criteria are met is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2334, "completion_tokens": 1062, "usd": 0.00552 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ffd4fd935f19", "task_id": "csv-already-clean", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (receipt parsing rules) but the task uses csv-cleaner skill; the diff is unrelated to CSV cleaning is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2324, "completion_tokens": 1133, "usd": 0.005723 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_699475671d46", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Both golden and replay runs produce identical cleaned CSV output meeting all success criteria: 3 data rows, 1 header, no duplicate IDs, no empty IDs, all valid original rows preserved. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2558, "completion_tokens": 1016, "usd": 0.005606 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_bbc5ac91d665", "task_id": "csv-payments", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run reports 'Rows in: 4' while the golden run reports 'Rows in: 5', but the input CSV has 5 data rows (wc -l shows 6 lines including header). This indicates the replay's cleaning logic incorrectly counted input rows is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2543, "completion_tokens": 1389, "usd": 0.00671 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_657e87d6ef5e", "task_id": "csv-header-only", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay meets all success criteria that the golden run meets, and both runs produce identical outputs for the empty input file is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2027, "completion_tokens": 1080, "usd": 0.005267 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_8d54d0935d9d", "task_id": "csv-attendees", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Both runs produce identical outputs and meet all success criteria: the cleaned file exists with 3 rows, no duplicates, no missing IDs, written to filesystem, and distinct from the original. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2397, "completion_tokens": 878, "usd": 0.005031 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_cbe81857d988", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.7, "prompt_tokens": 2445, "completion_tokens": 1649, "usd": 0.007392 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_db46f3b0746f", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output meets all success criteria: it formally states inability to deploy (using 'shipped' as a synonym), cites test failures as the reason, uses formal language, and contains no extra content is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 1368, "completion_tokens": 560, "usd": 0.003048 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_10a2feb40124", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1432, "completion_tokens": 1214, "usd": 0.005074 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_da322788c4ba", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff removes the explicit VAT line and merges VAT into the Total line, which changes the output format for expense reports is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1507, "completion_tokens": 1552, "usd": 0.006163 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.151487 }, "counterfactual_ultra_usd": 0.151487, "spent_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.153123 }, "prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 54666 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 52.754, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_732ed23735cd", "verdict_id": "verdict_732ed23735cd", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression" ], "headline": "Held: one judge found a regression on 6 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 6 of 13 replays. Review before merging. The agent ran 13 test pairs against the skill change, which only added a blank line to SKILL.md. One pair passed. Six pairs failed due to world changes where EUR/USD exchange rates shifted from 1.08 to 1.163, causing correct but different outputs. Six pairs regressed: three currency conversions and two summarization tasks now fail success criteria despite matching golden outputs, and one rewriting task uses a synonym. The judge flags these as regressions. The skill diff is cosmetic and cannot explain the logic failures. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_24416e3f8916", "task_id": "fx-no-date", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2207, "completion_tokens": 2117, "usd": 0.008558 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_c04a94e117fd", "task_id": "fx-weekend-date", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-11 was 1.0768 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2410, "completion_tokens": 804, "usd": 0.004822 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0768", "new_value": "1.1630" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_823b4ddae43a", "task_id": "fx-tokyo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2272, "completion_tokens": 745, "usd": 0.004507 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "0.0067", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_9326a185dca9", "task_id": "fx-paris", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output includes extra text 'Now produce final answer lines.' before the formatted lines, which is a cosmetic difference in the trace but not in the actual required output content is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.1, "prompt_tokens": 2456, "completion_tokens": 1269, "usd": 0.006263 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.1630" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_48d13ccbd36d", "task_id": "fx-zurich", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output, meeting all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.99, "prompt_tokens": 2241, "completion_tokens": 823, "usd": 0.00471 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.15714", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_24d29ab3083c", "task_id": "fx-london", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a blank line to SKILL.md, which is a cosmetic change that cannot affect functionality is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2433, "completion_tokens": 1658, "usd": 0.007407 } ], "control_outcome": null, "drift": [ { "fact_id": "t001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.2702", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_d540794f1c98", "task_id": "fx-coworking-courier", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2449, "completion_tokens": 1434, "usd": 0.006751 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ecc9e77802a1", "task_id": "fx-software", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "finance.rate returned 1.08 on 2026-10-14 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2234, "completion_tokens": 1077, "usd": 0.005465 } ], "control_outcome": null, "drift": [ { "fact_id": "obs_t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.08", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_ce1c4a74b4df", "task_id": "fx-hamburg-conference", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2713, "completion_tokens": 1378, "usd": 0.006847 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_84352baa167a", "task_id": "fx-berlin-trip", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2441, "completion_tokens": 957, "usd": 0.005312 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_68a265f4d0b3", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The replay output uses 'shipped' instead of 'deploy', which is a synonym and meets the criterion is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.0, "prompt_tokens": 1340, "completion_tokens": 395, "usd": 0.002525 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_eac0515b4a0c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1404, "completion_tokens": 963, "usd": 0.004293 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_aa0845d7fbd7", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly, meeting all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 742, "usd": 0.003633 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.071093 }, "counterfactual_ultra_usd": 0.071093, "spent_usd_by_tier": { "nano": 0.00068334, "super": 0.0020283, "ultra": 0.072443 }, "prompt_tokens_by_tier": { "nano": 8453, "super": 6227, "ultra": 28925 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 42.914, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_aa13a1b41910", "verdict_id": "verdict_aa13a1b41910", "state": "done", "decision": "BLOCK", "pipeline_decision": "WARN", "action": "held", "reasons": [ "single_judge_regression", "uncertain" ], "headline": "Held: one judge found a regression on 10 of 13 replays. Review before merging.", "narrative": "Held: one judge found a regression on 10 of 13 replays. Review before merging. The system replayed 13 test cases against a change that only added a trailing blank line to SKILL.md. One case passed, nine regressed, two drifted due to live exchange rate changes, and one was uncertain. The judge flagged regressions on cases where the replay output matched the golden output exactly, claiming the success criteria were no longer met despite identical results. Two cases failed because real-world FX rates shifted between runs. One case failed on a trivial capitalization difference. The uncertain case produced identical passing outputs but was held anyway. This pattern suggests the evaluation logic or judge is misaligned with the skill's actual behavior. Do not merge until the judge criteria are verified against the unchanged skill logic. Roll back if this is not intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_3e8797c16149", "task_id": "fx-no-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff adds a trailing blank line to SKILL.md, which is a cosmetic change that does not alter any step, tool call, or output is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2108, "completion_tokens": 618, "usd": 0.003962 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_608d63d15c36", "task_id": "fx-weekend-date", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff adds a trailing blank line to SKILL.md, which is a cosmetic change that does not alter any tool call, argument, or logic is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2433, "completion_tokens": 895, "usd": 0.005118 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_229dc213dbd6", "task_id": "fx-tokyo", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2307, "completion_tokens": 758, "usd": 0.004581 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "0.0067", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_e71800e8113a", "task_id": "fx-paris", "outcome": "drift", "expected_outcome": "pass", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-09 was 1.0768 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2401, "completion_tokens": 920, "usd": 0.005161 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_16ce4eaa1d1a", "task_id": "fx-zurich", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output is identical to the golden output and meets all success criteria is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2237, "completion_tokens": 674, "usd": 0.004259 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.15714", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_bd178338e4b1", "task_id": "fx-london", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The GBP/USD exchange rate on 2026-10-13 was 1.2702 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2436, "completion_tokens": 1725, "usd": 0.007611 } ], "control_outcome": null, "drift": [ { "fact_id": "t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.2702", "new_value": "1.3105" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_f6108436cd3f", "task_id": "fx-coworking-courier", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff added a trailing blank line to SKILL.md, which is a cosmetic formatting change that does not alter any step, tool call, argument, or rule is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2442, "completion_tokens": 759, "usd": 0.004719 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_133f03cc5707", "task_id": "fx-software", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Both golden and replay runs produce identical outputs meeting all success criteria. Held for a human.", "opinions": [ { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2249, "completion_tokens": 668, "usd": 0.004253 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_cb6d6a6fa18e", "task_id": "fx-hamburg-conference", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a blank line to SKILL.md, which is a cosmetic change that cannot affect runtime behavior is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2754, "completion_tokens": 891, "usd": 0.005427 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_9bc309eb22db", "task_id": "fx-berlin-trip", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output uses lowercase 'taxi' and 'hotel' while the golden output uses capitalized 'Taxi' and 'Hotel'; this is a cosmetic difference in capitalization and does not affect the required values or structure is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2476, "completion_tokens": 1264, "usd": 0.006268 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": false, "status": "checked", "attributable": null, "old_value": "1.0800", "new_value": null } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_c0d90d4cb42f", "task_id": "canary-formal-rewrite", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff adds a trailing blank line to the skill file, which changes the prompt sent to the model and could affect output formatting or content is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 754, "usd": 0.003602 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_7d778bb3b477", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1404, "completion_tokens": 1699, "usd": 0.006501 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9620dc2201ff", "task_id": "canary-summary", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output matches the golden output exactly is no longer met.", "opinions": [ { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 1222, "usd": 0.005073 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0, "super": 0.0, "ultra": 0.066535 }, "counterfactual_ultra_usd": 0.066535, "spent_usd_by_tier": { "nano": 0.00045348, "super": 0.0019689, "ultra": 0.067885 }, "prompt_tokens_by_tier": { "nano": 6362, "super": 6038, "ultra": 28819 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 47.442, "simulated_models": false, "repeat": 0, "error": null } ] }, "drift_off": { "config": { "name": "drift_off", "description": "Full cascade without Tavily drift disambiguation.", "experiments": [ "E3" ], "mode": "edits", "cascade": { "mode": "cascade", "level0": true, "drift": false, "controls": true }, "single_judge_blocks": null, "types": [ "R1", "R2", "R3", "R4", "R5", "R6", "N0", "D1" ], "noise_repeats": 3, "runner": "local", "concurrency": 2, "reuse_replays": "cascade", "max_traces": 40 }, "config_path": "bench/configs/drift_off.yaml", "command": "ratchet bench run --config bench/configs/drift_off.yaml --out bench/results/2026-10-16 --types D1 --limit 2 --yes", "started_at": "2026-10-16T13:00:18.558437Z", "updated_at": "2026-10-16T13:58:42.384973Z", "edits": [ { "bench_id": "n0-github-triage-2", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_7ddd87ef70ac", "verdict_id": "verdict_7ddd87ef70ac", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The skill diff only rewords \"when\" to \"whenever\" and \"Steps\" to \"Procedure\" without changing logic. Eleven replays passed, but two were held for human review: a triage case where the diff did not explain the behavioral difference, and a math case attributed to likely nondeterminism. No regressions, drift, or improvements were detected. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_811b35141f2c", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 99, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_b3d50ce1b799", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1217, "completion_tokens": 138, "usd": 0.00010614 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_7741c40ef280", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 131, "usd": 0.00011274 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_abe3d0283a76", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 138, "usd": 0.0001155 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_a0217d5141b7", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 167, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_f5d663dbc019", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only rewords 'when' to 'whenever' and 'Steps' to 'Procedure' without changing any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1383, "completion_tokens": 149, "usd": 0.00011874 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3327, "completion_tokens": 299, "usd": 0.0012672 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3272, "completion_tokens": 2684, "usd": 0.011324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7e7f0890e15f", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1370, "completion_tokens": 159, "usd": 0.00012036 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2975, "completion_tokens": 345, "usd": 0.001203 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a1def1b764dc", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 131, "usd": 0.0001122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_0490537a6338", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 171, "usd": 0.00012444 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_8b534e552950", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 135, "usd": 0.00011526 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_7e792837459f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 87, "usd": 0.00008916 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_95212662e369", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 68, "usd": 0.00008508 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1573, "completion_tokens": 217, "usd": 0.0006672 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1446, "completion_tokens": 1239, "usd": 0.005163 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a24da7604ca9", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 143, "usd": 0.00010488 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1600, "completion_tokens": 220, "usd": 0.000678 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00141726, "super": 0.0038154, "ultra": 0.016487 }, "counterfactual_ultra_usd": 0.05111, "spent_usd_by_tier": { "nano": 0.00141726, "super": 0.0038154, "ultra": 0.017178 }, "prompt_tokens_by_tier": { "nano": 16757, "super": 9475, "ultra": 5070 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 17.027, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-2", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_a66c3a899d39", "verdict_id": "verdict_a66c3a899d39", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 22 replays. Review before merging. The system ran 22 regression pairs against the expense-report skill change; 19 passed but 3 were held as uncertain. The uncertain cases cover filing ink and fuel receipts, cleaning signups.csv, and computing a percentage. In each case the judge could not verify the output matched expectations, citing likely nondeterminism or insufficient evidence in the diff. No regressions, drift, or improvements were detected. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_db2fa52f33ec", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1106, "completion_tokens": 121, "usd": 0.0000954 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_663c04fc7679", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 192, "usd": 0.00013182 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_b8eacf7ca9eb", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Criteria 1-5 are met: t003 shows both receipts appended with correct merchants, dates, totals, VAT amounts, and grand total. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1637, "completion_tokens": 130, "usd": 0.00012942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3999, "completion_tokens": 154, "usd": 0.0013383 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3945, "completion_tokens": 2836, "usd": 0.012453 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_df4a9a6d2d5e", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 131, "usd": 0.00010806 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_2aba1c6b9a48", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 83, "usd": 0.00010566 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_4d8a32096183", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 158, "usd": 0.00012444 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_5d79f77bf29a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1315, "completion_tokens": 164, "usd": 0.00011826 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_fa5cfd551a14", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 141, "usd": 0.00011232 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_bcd2f6c4de1b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 169, "usd": 0.00013182 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6c9bfaed6179", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 199, "usd": 0.0001353 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_979bc36816b3", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 150, "usd": 0.00010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_0444b760bf07", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 152, "usd": 0.00011274 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_997e188d6e47", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 148, "usd": 0.00011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_d40c1503ab73", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 147, "usd": 0.00011088 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_8111d388d469", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 183, "usd": 0.00012096 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0727abbc4d2a", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 294, "usd": 0.00014472 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2772, "completion_tokens": 294, "usd": 0.0010962 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_cfff2f4aa409", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 160, "usd": 0.00011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_52606712bd67", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 198, "usd": 0.00012342 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_542487f91fa5", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 102, "usd": 0.00009942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2698, "completion_tokens": 334, "usd": 0.00111 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2668, "completion_tokens": 3771, "usd": 0.013981 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_599436fdbbc6", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 121, "usd": 0.00009804 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_de6195626ea9", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1122, "completion_tokens": 72, "usd": 0.0000846 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1722, "completion_tokens": 221, "usd": 0.0007155 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1580, "completion_tokens": 962, "usd": 0.004466 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e212cacce84", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 109, "usd": 0.00009738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00251676, "super": 0.00426, "ultra": 0.0309 }, "counterfactual_ultra_usd": 0.083722, "spent_usd_by_tier": { "nano": 0.00251676, "super": 0.00426, "ultra": 0.031653 }, "prompt_tokens_by_tier": { "nano": 28650, "super": 11191, "ultra": 8580 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 25.226, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_ad4fa9f6a3f6", "verdict_id": "verdict_ad4fa9f6a3f6", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The system replayed 13 test cases against the edited fx-expense-converter skill; 11 passed, 2 were uncertain, and none regressed or drifted. One uncertain case produced a rate date of 2026-10-09 instead of the expected 2026-10-11. The other showed a numeric difference the skill diff does not explain, suggesting nondeterminism. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_885b30526626", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1347, "completion_tokens": 224, "usd": 0.00013458 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2512, "completion_tokens": 322, "usd": 0.0010434 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_fcf301073639", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay shows the rate date as 2026-10-09 in the final output while the golden shows 2026-10-11, violating the 'uses the correct date 2026-10-11' criterion. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1378, "completion_tokens": 120, "usd": 0.00011148 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2758, "completion_tokens": 279, "usd": 0.0010785 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2716, "completion_tokens": 2368, "usd": 0.00982 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_82a70ccb2af8", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1351, "completion_tokens": 180, "usd": 0.00012426 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2612, "completion_tokens": 303, "usd": 0.0010563 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_973ca2670ff2", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1405, "completion_tokens": 89, "usd": 0.00010566 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2734, "completion_tokens": 322, "usd": 0.00111 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3afca948a379", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 191, "usd": 0.0001308 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2573, "completion_tokens": 339, "usd": 0.001077 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_f5be008e5f9d", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 195, "usd": 0.00013122 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2751, "completion_tokens": 332, "usd": 0.0011241 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_b55b161ede07", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1474, "completion_tokens": 181, "usd": 0.00013188 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2758, "completion_tokens": 343, "usd": 0.0011361 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_e70094e8923d", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1354, "completion_tokens": 197, "usd": 0.00012852 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2561, "completion_tokens": 310, "usd": 0.0010473 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5bde00727c59", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1475, "completion_tokens": 123, "usd": 0.00011802 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2993, "completion_tokens": 307, "usd": 0.0011742 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_96be56214418", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1435, "completion_tokens": 189, "usd": 0.00013146 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2746, "completion_tokens": 309, "usd": 0.0011019 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d1a68d58b4ae", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 157, "usd": 0.00010668 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1671, "completion_tokens": 222, "usd": 0.0007011 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_96c18d79778a", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 80, "usd": 0.0000894 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1723, "completion_tokens": 221, "usd": 0.0007158 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1585, "completion_tokens": 1734, "usd": 0.006787 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9cb3c3858cd1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 121, "usd": 0.00010098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00154494, "super": 0.0123657, "ultra": 0.016607 }, "counterfactual_ultra_usd": 0.081528, "spent_usd_by_tier": { "nano": 0.00154494, "super": 0.0123657, "ultra": 0.017365 }, "prompt_tokens_by_tier": { "nano": 17561, "super": 30392, "ultra": 4678 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.087, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-1", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_eb8701615518", "verdict_id": "verdict_eb8701615518", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now fails \"states that no details were requested\" on 5 of 13 past tasks.", "narrative": "Blocked: github-triage now fails \"states that no details were requested\" on 5 of 13 past tasks. The edit changed SKILL.md and triggered a replay of 13 evaluation pairs. Four passed, five regressed, and four were held for human review due to judge disagreement or ambiguous success criteria. The five regressions all involve the agent failing to ask for required issue details on tasks like acme/data-pipeline, acme/docs-site, acme/mobile-app, acme/billing-api, and acme/widgets. The uncertain pairs include labeling disagreements and a math task where judges conflicted. The change was automatically reverted. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_29198b3e615a", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 155, "usd": 0.00010554 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_ae5049772f22", "task_id": "tri-status-page", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "Judges disagreed: Super found a regression and Ultra did not. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1175, "completion_tokens": 130, "usd": 0.0001017 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2395, "completion_tokens": 220, "usd": 0.0009165 }, { "tier": "ultra", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2340, "completion_tokens": 4684, "usd": 0.016392 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_6a4a7b2c3a20", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails the success criterion 'labels issue #502 as question' (it labels it 'enhancement' instead). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1365, "completion_tokens": 127, "usd": 0.00011238 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3185, "completion_tokens": 346, "usd": 0.0012669 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3102, "completion_tokens": 3004, "usd": 0.012114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ff7547adf9e1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1364, "completion_tokens": 173, "usd": 0.00012336 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_95dc2c9a2266", "task_id": "tri-auth-service", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets all six success criteria: it states 4 open issues, labels #88 bug, #90 enhancement, #91 question, #92 already labeled, and asks for details on #91 via the comment in step t005. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1286, "completion_tokens": 130, "usd": 0.00010836 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2979, "completion_tokens": 249, "usd": 0.0011178 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.6, "prompt_tokens": 2907, "completion_tokens": 3375, "usd": 0.013032 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_1ac70e25ffb3", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states that no details were requested is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1353, "completion_tokens": 150, "usd": 0.00011718 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3174, "completion_tokens": 302, "usd": 0.001224 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3057, "completion_tokens": 3487, "usd": 0.013518 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_3be8109cfb2f", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states that no details were asked for is no longer met.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 151, "usd": 0.0001194 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3134, "completion_tokens": 263, "usd": 0.0011769 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3083, "completion_tokens": 931, "usd": 0.005876 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_650e3f7e3ede", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "asks for details on #64 and cites github.comment as artifact changed is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1308, "completion_tokens": 186, "usd": 0.00012312 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3018, "completion_tokens": 261, "usd": 0.0011403 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3034, "completion_tokens": 1959, "usd": 0.008911 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_e5bee22bc04e", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "mentions that issue #36 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1400, "completion_tokens": 96, "usd": 0.00010704 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3411, "completion_tokens": 294, "usd": 0.0012879 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3260, "completion_tokens": 1663, "usd": 0.008249 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_0c922bfa825f", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "mentions #15 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 87, "usd": 0.00010392 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3130, "completion_tokens": 302, "usd": 0.0012108 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3009, "completion_tokens": 1219, "usd": 0.006666 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_00c8cee98bc3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1138, "completion_tokens": 126, "usd": 0.00009852 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_05517158aa56", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output contains the numeric result 40.8, the phrase '17% of 240', and the phrase '= 0.17 × 240' as shown in step m001. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1557, "completion_tokens": 219, "usd": 0.0006642 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 1445, "completion_tokens": 1222, "usd": 0.005111 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_bdbe9648c85c", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 116, "usd": 0.00009978 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00140706, "super": 0.0100053, "ultra": 0.089869 }, "counterfactual_ultra_usd": 0.144972, "spent_usd_by_tier": { "nano": 0.00140706, "super": 0.0100053, "ultra": 0.090949 }, "prompt_tokens_by_tier": { "nano": 16655, "super": 25983, "ultra": 25903 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 34.575, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-1", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_89856d9c076d", "verdict_id": "verdict_89856d9c076d", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail", "agreed_regression" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system replayed 13 golden cases against the proposed change; 10 passed, 2 regressed, and 1 was uncertain. The clearest regression is acme/docs-site, where the agent made a github.comment call the original run never made and the sandbox rejected it. A second regression in acme/status-page dropped a required output constraint. The uncertain case acme/data-pipeline produced only three label calls instead of six. Because the change introduces a new side effect and breaks existing behavior, the edit was reverted automatically. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_165175858316", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 92, "usd": 0.00009186 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_187d5cdcf6dc", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "contains no additional text that is not part of the required output format is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1228, "completion_tokens": 136, "usd": 0.00010632 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2602, "completion_tokens": 227, "usd": 0.0009849 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2568, "completion_tokens": 1044, "usd": 0.0057 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3f0a0fcb0075", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 138, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_9972d203f927", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1361, "completion_tokens": 137, "usd": 0.00011454 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c71e5c2370b8", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 129, "usd": 0.00010848 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3017, "completion_tokens": 287, "usd": 0.0011634 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_eb9d9e6851a5", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay run produces only 3 github.label calls (t002-t004) while the golden run produces 6 (t002-t007), failing the 'exactly six github.label changes' criterion. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 188, "usd": 0.00012816 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3101, "completion_tokens": 328, "usd": 0.0012255 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3116, "completion_tokens": 4398, "usd": 0.01631 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_de7759ea9e17", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_092b5ebe7778", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1350, "completion_tokens": 183, "usd": 0.00012492 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3084, "completion_tokens": 289, "usd": 0.0011853 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_968755dee57c", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1413, "completion_tokens": 168, "usd": 0.0001251 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_10dca42d0cf4", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1392, "completion_tokens": 127, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_990c76131fc0", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_47c34378e66c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1182, "completion_tokens": 134, "usd": 0.00010308 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_602f5ee8bbac", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 82, "usd": 0.00009234 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00131676, "super": 0.0045591, "ultra": 0.02201 }, "counterfactual_ultra_usd": 0.057541, "spent_usd_by_tier": { "nano": 0.00131676, "super": 0.0045591, "ultra": 0.022829 }, "prompt_tokens_by_tier": { "nano": 15498, "super": 11804, "ultra": 6095 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 20.513, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-1", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_3e723adb08e5", "verdict_id": "verdict_3e723adb08e5", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail", "agreed_regression" ], "headline": "Blocked: the new github-triage would close issues in 5 of 13 replays.", "narrative": "Blocked: the new github-triage would close issues in 5 of 13 replays. The system replayed 13 test pairs against the edited skill, comparing every GitHub API call against the original golden runs. Six passed exactly. Five regressed by making new calls to `github.close_issue` or `github.comment` that the sandbox rejected. One triage case drifted, producing only three labels instead of the required six. A math regression also appeared where the output phrase for 17% of 240 changed. The edit introduced unsafe side effects across multiple repositories and broke a basic calculation. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_f8c354ae33e9", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 100, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_478d9ee2ce19", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 151, "usd": 0.00010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_755deee2dac3", "task_id": "tri-search-indexer", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ce4b9acb3157", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 3 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 3, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_11ee7c7f4f30", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_9432a20fd19d", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay run makes only 3 github.label calls versus 6 in the golden run, failing the 'exactly six github.label changes' success criterion. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1419, "completion_tokens": 154, "usd": 0.0001221 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3381, "completion_tokens": 279, "usd": 0.0012654 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3291, "completion_tokens": 5159, "usd": 0.018768 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_715dfb3b7eec", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_5ba0f0c4e92d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_39066d273f98", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1415, "completion_tokens": 191, "usd": 0.00013074 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3566, "completion_tokens": 336, "usd": 0.0013722 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_c944d9078bcc", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1396, "completion_tokens": 129, "usd": 0.00011472 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3344, "completion_tokens": 342, "usd": 0.001311 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_56c2b63686fb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 97, "usd": 0.00009372 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1600, "completion_tokens": 247, "usd": 0.0007023 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a8e17ca9255f", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "output includes the phrase '= **40.8**' is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 143, "usd": 0.0001038 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1616, "completion_tokens": 219, "usd": 0.0006819 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1542, "completion_tokens": 1290, "usd": 0.005412 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_dd5897c02420", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 98, "usd": 0.00009474 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00085926, "super": 0.0053328, "ultra": 0.02418 }, "counterfactual_ultra_usd": 0.055214, "spent_usd_by_tier": { "nano": 0.00085926, "super": 0.0053328, "ultra": 0.025179 }, "prompt_tokens_by_tier": { "nano": 10069, "super": 13507, "ultra": 5436 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 21.88, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_18141231cc37", "verdict_id": "verdict_18141231cc37", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 4 of 14 replays. Review before merging.", "narrative": "Held: the judges were not sure on 4 of 14 replays. Review before merging. The skill diff only rewords the description with no functional change. Ten replays passed. Four were held: a weekend conversion and a receipt filing blocked by an external Gmail restriction showed unclear results, while a formal rewrite and a math calculation appeared to be nondeterministic. No regressions, drifts, or improvements were found. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_9b9fe0986841", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1324, "completion_tokens": 129, "usd": 0.0001104 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_f97cae02fbc4", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only rewords the description from 'foreign-currency amounts' to 'foreign-currency expense amounts' with no functional change. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1352, "completion_tokens": 171, "usd": 0.00012216 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2555, "completion_tokens": 283, "usd": 0.0010212 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2574, "completion_tokens": 1833, "usd": 0.008073 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6eb01a2d41d3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1342, "completion_tokens": 121, "usd": 0.00010956 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_315104ec0683", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 93, "usd": 0.00010524 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_762ca612dbe8", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 216, "usd": 0.00013326 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_4ef2ca8f8b72", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1396, "completion_tokens": 213, "usd": 0.00013488 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_53e1872e54f8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 135, "usd": 0.0001188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_fddd61f372fe", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 151, "usd": 0.00011538 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_bda2112c7543", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 129, "usd": 0.00011808 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c3051db65b47", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 159, "usd": 0.00012288 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_8981ccff882f", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay fails the email-sent criterion because gmail.send was blocked by 'ratchet_blocked', a system restriction external to the skill. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1650, "completion_tokens": 151, "usd": 0.00013524 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3818, "completion_tokens": 162, "usd": 0.0012912 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3805, "completion_tokens": 1309, "usd": 0.007732 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_ec291cce2b97", "task_id": "canary-formal-rewrite", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 100, "usd": 0.00009324 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1514, "completion_tokens": 266, "usd": 0.0006936 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1502, "completion_tokens": 964, "usd": 0.004394 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_cbb67ab2491c", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1533, "completion_tokens": 214, "usd": 0.0006525 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1389, "completion_tokens": 1278, "usd": 0.005223 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_91fcb24262ed", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00159822, "super": 0.0036585, "ultra": 0.025422 }, "counterfactual_ultra_usd": 0.062332, "spent_usd_by_tier": { "nano": 0.00159822, "super": 0.0036585, "ultra": 0.026157 }, "prompt_tokens_by_tier": { "nano": 18949, "super": 9420, "ultra": 9711 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.151, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_8f88351ea240", "verdict_id": "verdict_8f88351ea240", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs for the fx-expense-converter skill after a documentation-only change to SKILL.md. Twelve pairs passed and one was marked uncertain. The uncertain case, \"Compute 17% of 240,\" produced output that appears to meet all success criteria, including the correct numeric result and required phrasing, but the judge withheld automatic approval. No regressions, drift, or improvements were detected. Review the uncertain replay to confirm the output is acceptable. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_382f2e4d44ae", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1300, "completion_tokens": 129, "usd": 0.00010896 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_75e2cc6a76a8", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 160, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_c536f0cd0389", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 157, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_a8c493d9890b", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 100, "usd": 0.00010692 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_b7a6a78b225c", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1345, "completion_tokens": 203, "usd": 0.00012942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a00a986b53f7", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1383, "completion_tokens": 130, "usd": 0.00011418 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_c7bcfb7bf68e", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 206, "usd": 0.00013512 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_3b832d8b5c25", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1367, "completion_tokens": 164, "usd": 0.00012138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_f76c8a253ef0", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 123, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c432f688c91f", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 136, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1f1fa89dddfb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_9470f1a93d4f", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output meets all success criteria: it contains the numeric result 40.8, the phrase '17% of 240', the phrase '= 0.17 × 240', and the value 40.8 after an equals sign;. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 72, "usd": 0.00008532 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1592, "completion_tokens": 254, "usd": 0.0007062 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1487, "completion_tokens": 1120, "usd": 0.004847 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f86b4f70f4f6", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 96, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00145608, "super": 0.0007062, "ultra": 0.004847 }, "counterfactual_ultra_usd": 0.029695, "spent_usd_by_tier": { "nano": 0.00145608, "super": 0.0007062, "ultra": 0.00559 }, "prompt_tokens_by_tier": { "nano": 17172, "super": 1592, "ultra": 1831 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 11.258, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-1", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_428f769a0e2a", "verdict_id": "verdict_428f769a0e2a", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: expense-report now fails \"signups.csv contains exactly 4 rows before cleaning\" on 1 of 22 past tasks.", "narrative": "Blocked: expense-report now fails \"signups.csv contains exactly 4 rows before cleaning\" on 1 of 22 past tasks. The evaluation ran 22 previous tasks against the edited skill, finding 18 pass, 1 regression, 3 uncertain, and 0 improved. The regression on \"Clean signups.csv\" shows the output no longer contains exactly four rows before cleaning. The three uncertain results involve an email not sent to finance, a ticket cleaning mismatch, and a math calculation difference, all held for human review due to likely nondeterminism or unclear causes. The skill diff shows only a one-line change in SKILL.md. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_ddb84b0fe47b", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1178, "completion_tokens": 117, "usd": 0.00009876 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1647, "completion_tokens": 154, "usd": 0.0006327 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_7d67f2353f20", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 192, "usd": 0.0001305 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3166, "completion_tokens": 315, "usd": 0.0012333 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_acc26084d722", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails the criterion requiring an email to be sent to with the expense report (t004 blocked: ratchet_blocked). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1658, "completion_tokens": 135, "usd": 0.00013188 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3971, "completion_tokens": 134, "usd": 0.0013119 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3941, "completion_tokens": 2073, "usd": 0.01016 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_af5ade88f377", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 172, "usd": 0.00011934 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2541, "completion_tokens": 271, "usd": 0.0010062 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_a69ed051d7f4", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1441, "completion_tokens": 141, "usd": 0.0001203 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3266, "completion_tokens": 296, "usd": 0.0012462 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_3e7d7797755d", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1430, "completion_tokens": 195, "usd": 0.0001326 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3239, "completion_tokens": 305, "usd": 0.0012462 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_079c36312676", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1303, "completion_tokens": 170, "usd": 0.00011898 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2566, "completion_tokens": 273, "usd": 0.0010155 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_f07720304e5f", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1296, "completion_tokens": 130, "usd": 0.00010896 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2555, "completion_tokens": 273, "usd": 0.0010122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_8626f6fe5abb", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1521, "completion_tokens": 117, "usd": 0.00011934 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3365, "completion_tokens": 383, "usd": 0.0013542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_521e59e6091f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1435, "completion_tokens": 173, "usd": 0.00012762 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3257, "completion_tokens": 310, "usd": 0.0012561 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_e31167be7710", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 163, "usd": 0.00011166 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_229e9b0e57cc", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 149, "usd": 0.00011202 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_1aec4a330ef6", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 180, "usd": 0.0001179 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_5b8721009163", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 182, "usd": 0.00011802 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3aee4f6f9c1d", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 135, "usd": 0.00010944 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2742, "completion_tokens": 280, "usd": 0.0010746 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2634, "completion_tokens": 2567, "usd": 0.010335 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_838608cb3f0e", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 127, "usd": 0.00010752 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2693, "completion_tokens": 285, "usd": 0.0010644 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_010f6ae26a1d", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 182, "usd": 0.00011934 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2252, "completion_tokens": 265, "usd": 0.0009141 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_a8ba615a5d23", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1313, "completion_tokens": 162, "usd": 0.00011766 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_22e0e0ff85d9", "task_id": "csv-signups", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "signups.csv contains exactly 4 rows before cleaning is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 180, "usd": 0.00011814 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2774, "completion_tokens": 339, "usd": 0.0011373 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2729, "completion_tokens": 4212, "usd": 0.015365 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_15046a030360", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3ac334e2f481", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1621, "completion_tokens": 286, "usd": 0.0007437 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1567, "completion_tokens": 2136, "usd": 0.007975 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb845c98bd90", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1175, "completion_tokens": 102, "usd": 0.00009498 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00251472, "super": 0.0162486, "ultra": 0.043835 }, "counterfactual_ultra_usd": 0.136633, "spent_usd_by_tier": { "nano": 0.00251472, "super": 0.0162486, "ultra": 0.04468 }, "prompt_tokens_by_tier": { "nano": 28808, "super": 41655, "ultra": 11296 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 30.944, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-expense-report-1", "skill": "expense-report", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_e265451a334f", "verdict_id": "verdict_e265451a334f", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 7 of 22 past tasks.", "narrative": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 7 of 22 past tasks. The edit changed SKILL.md and triggered regressions across seven receipt-filing tasks where structured extraction failed completely, plus a math task losing its formatted output. Two tasks are uncertain: one blocked on a gmail tool call and one held despite passing criteria. Thirteen tasks still pass. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_73fd927ba82e", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 103, "usd": 0.00009396 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1665, "completion_tokens": 142, "usd": 0.0006273 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_71599b6cf2f5", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1399, "completion_tokens": 233, "usd": 0.00013986 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3335, "completion_tokens": 279, "usd": 0.0012516 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3295, "completion_tokens": 2245, "usd": 0.01003 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_68029dbf8d42", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails the success criterion requiring the email to be sent to with the grand total, because the gmail.send tool was blocked with 'ratchet_blocked' (t004). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1621, "completion_tokens": 128, "usd": 0.00012798 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3802, "completion_tokens": 401, "usd": 0.0015015 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3792, "completion_tokens": 3295, "usd": 0.013677 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_0061479721a1", "task_id": "exp-software", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the VAT 18.81 EUR is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1293, "completion_tokens": 91, "usd": 0.00009942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2488, "completion_tokens": 269, "usd": 0.0009885 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2364, "completion_tokens": 814, "usd": 0.004806 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_cde84ba8b5d5", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1427, "completion_tokens": 191, "usd": 0.00013146 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3176, "completion_tokens": 362, "usd": 0.0012786 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_d2bfe3e90095", "task_id": "exp-dinner-parking", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 109, "usd": 0.00011106 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3180, "completion_tokens": 307, "usd": 0.0012303 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3108, "completion_tokens": 1024, "usd": 0.00618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_ea8d1c2b4f4b", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 150, "usd": 0.00011304 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_bcff8a71d40d", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the VAT as 79.83 EUR is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1312, "completion_tokens": 137, "usd": 0.0001116 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2525, "completion_tokens": 275, "usd": 0.001005 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2462, "completion_tokens": 1301, "usd": 0.006365 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_49a1a41ef5ee", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1507, "completion_tokens": 102, "usd": 0.0001149 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3382, "completion_tokens": 384, "usd": 0.0013602 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.98, "prompt_tokens": 3214, "completion_tokens": 1562, "usd": 0.0079 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_87ffd7a9b61f", "task_id": "exp-berlin-trip", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1444, "completion_tokens": 152, "usd": 0.00012312 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3186, "completion_tokens": 334, "usd": 0.0012564 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.98, "prompt_tokens": 3139, "completion_tokens": 1298, "usd": 0.007033 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_bfab509e3676", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1257, "completion_tokens": 157, "usd": 0.0001131 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_ae36c7d648f7", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 107, "usd": 0.00009978 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_c94efe16dd1d", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 106, "usd": 0.00010014 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ffd4fd935f19", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1263, "completion_tokens": 137, "usd": 0.00010866 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_699475671d46", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 222, "usd": 0.0001296 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_bbc5ac91d665", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 112, "usd": 0.0001032 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2718, "completion_tokens": 306, "usd": 0.0010908 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_657e87d6ef5e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 258, "usd": 0.00013758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_8d54d0935d9d", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 190, "usd": 0.00012294 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_cbe81857d988", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "All six success criteria are satisfied in the replay run: the input file has 4 data rows (t001, t002), one row with empty id (t001), the output has 3 rows with no empty ids and no duplicates (t003), and signups.clean.csv is created by overw…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 123, "usd": 0.00010374 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2578, "completion_tokens": 327, "usd": 0.0010677 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.6, "prompt_tokens": 2486, "completion_tokens": 2099, "usd": 0.008783 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_db46f3b0746f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 109, "usd": 0.00009444 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1587, "completion_tokens": 221, "usd": 0.000675 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_10a2feb40124", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "output includes the phrase \"= **40.8**\" is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 137, "usd": 0.00010236 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1627, "completion_tokens": 214, "usd": 0.0006807 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1587, "completion_tokens": 1128, "usd": 0.004971 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_da322788c4ba", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00247356, "super": 0.0140136, "ultra": 0.069745 }, "counterfactual_ultra_usd": 0.154547, "spent_usd_by_tier": { "nano": 0.00247356, "super": 0.0140136, "ultra": 0.070725 }, "prompt_tokens_by_tier": { "nano": 28682, "super": 35249, "ultra": 26139 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 26.668, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-2", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_f5d839606056", "verdict_id": "verdict_f5d839606056", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 2 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 2 of 13 replays. The system replayed 13 test cases against the proposed change to SKILL.md. Eight passed. Two regressed because the agent made new github.comment calls that the sandbox rejected. Three are uncertain: one shows fewer label operations than the golden run, one omits a required issue reference, and one is a math task unrelated to the skill change. The regressions prove the edit introduces unsafe side effects. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5bbf428edf52", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 115, "usd": 0.00009564 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1606, "completion_tokens": 177, "usd": 0.0006411 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f2d56880f7aa", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 152, "usd": 0.00010878 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_c651c351049c", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1384, "completion_tokens": 146, "usd": 0.00011808 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_b6ff0cb2781b", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "pass", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac63d98c8094", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1314, "completion_tokens": 140, "usd": 0.00011244 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_537a1694bbb0", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output textually meets all success criteria (criteria 1-4 at m004, criterion 5 partially via t002-t004), but the tool trace shows only three github.label write_reversible calls (t002-t004) versus six in the golden run (t002-t007)…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1395, "completion_tokens": 148, "usd": 0.00011922 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3372, "completion_tokens": 312, "usd": 0.0012924 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3301, "completion_tokens": 3208, "usd": 0.012925 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_9e2fa3314653", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d200a62ffe51", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 153, "usd": 0.00011748 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_465ab5c43f9c", "task_id": "tri-billing-api", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay violates the criterion 'mentions that issue #36 was asked for details' (replay step m003 says 'Asked for details: none' vs golden's 'Asked for details: #36'). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1401, "completion_tokens": 110, "usd": 0.00011046 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3357, "completion_tokens": 329, "usd": 0.0013032 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3327, "completion_tokens": 4337, "usd": 0.016338 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_19dcfee12b9c", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 140, "usd": 0.0001176 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f59a42a6f25f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f5acfa748023", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies a github triage skill (adding a branch parameter) but the task is a simple percentage calculation unrelated to that skill. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1626, "completion_tokens": 222, "usd": 0.0006876 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1502, "completion_tokens": 2795, "usd": 0.009887 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_de28baa02c6f", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1211, "completion_tokens": 119, "usd": 0.00010122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0011802, "super": 0.0039243, "ultra": 0.03915 }, "counterfactual_ultra_usd": 0.070508, "spent_usd_by_tier": { "nano": 0.0011802, "super": 0.0039243, "ultra": 0.040054 }, "prompt_tokens_by_tier": { "nano": 14098, "super": 9961, "ultra": 8707 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 20.227, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-2", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_75e4ac298b54", "verdict_id": "verdict_75e4ac298b54", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The skill change modified github-triage SKILL.md. Eleven replays passed, but two were held for human review. In \"Triage acme/mobile-app,\" the replay failed to label issue #58 as a bug and incorrectly claimed it was already labeled. In \"Compute 17% of 240,\" the output differed from the baseline without a clear explanation in the skill diff, suggesting nondeterminism. No regressions, drift, or improvements were detected. Costs were minimal across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0a31fbd0f6ab", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 114, "usd": 0.0000957 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e3ade3806218", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1185, "completion_tokens": 132, "usd": 0.00010278 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e7ec930d750a", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1359, "completion_tokens": 165, "usd": 0.00012114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_c0c9ec6a7dc8", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1353, "completion_tokens": 119, "usd": 0.00010974 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_9484d74e29ab", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1330, "completion_tokens": 147, "usd": 0.00011508 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3022, "completion_tokens": 252, "usd": 0.0011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_dcfc05cb8a6e", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1403, "completion_tokens": 160, "usd": 0.00012258 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3125, "completion_tokens": 396, "usd": 0.0012939 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_b752b4bba738", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 167, "usd": 0.00012546 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3006, "completion_tokens": 258, "usd": 0.001134 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_811329a74d4f", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails to label #58 as bug (criterion 2), instead incorrectly listing it as already labeled. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1353, "completion_tokens": 111, "usd": 0.00010782 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2973, "completion_tokens": 292, "usd": 0.0011547 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2831, "completion_tokens": 2013, "usd": 0.00887 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9dd1c1c9ae58", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 174, "usd": 0.00012462 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3625, "completion_tokens": 295, "usd": 0.001353 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d9834ffa3e3a", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1463, "completion_tokens": 110, "usd": 0.00011418 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3343, "completion_tokens": 294, "usd": 0.0012675 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3f0ff18aec4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 119, "usd": 0.00009756 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1497, "completion_tokens": 213, "usd": 0.0006408 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_5b3dba8d5830", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1537, "completion_tokens": 219, "usd": 0.0006582 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1446, "completion_tokens": 1885, "usd": 0.007101 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f28450be2e37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 119, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142464, "super": 0.0086355, "ultra": 0.015971 }, "counterfactual_ultra_usd": 0.066791, "spent_usd_by_tier": { "nano": 0.00142464, "super": 0.0086355, "ultra": 0.016724 }, "prompt_tokens_by_tier": { "nano": 16908, "super": 22128, "ultra": 4619 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.313, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_f650857d26c3", "verdict_id": "verdict_f650857d26c3", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs against the updated fx-expense-converter skill and found 12 passing with no regressions or drift. One case, \"Compute 17% of 240,\" produced an uncertain result because the skill change does not explain the output difference, suggesting possible nondeterminism. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_bf3fa0074e07", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 129, "usd": 0.00010968 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_e649669b0a50", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 175, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6b67a590cae3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1340, "completion_tokens": 153, "usd": 0.00011712 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_1f156a1c8f92", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1370, "completion_tokens": 92, "usd": 0.00010428 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3e98cef87a9d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 229, "usd": 0.00013638 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9ffedc5fd087", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1372, "completion_tokens": 126, "usd": 0.00011256 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_941456f71c6b", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1439, "completion_tokens": 156, "usd": 0.00012378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_4edc319d8099", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 177, "usd": 0.00012378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5ece1e8c9118", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 118, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_400354316051", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1388, "completion_tokens": 141, "usd": 0.00011712 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_6cfa1d232eb4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_13feb5997c98", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 73, "usd": 0.00008772 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1543, "completion_tokens": 218, "usd": 0.0006591 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1410, "completion_tokens": 2086, "usd": 0.007668 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e2f6c6c99b4", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 105, "usd": 0.00009714 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00145704, "super": 0.0006591, "ultra": 0.007668 }, "counterfactual_ultra_usd": 0.032377, "spent_usd_by_tier": { "nano": 0.00145704, "super": 0.0006591, "ultra": 0.008264 }, "prompt_tokens_by_tier": { "nano": 17196, "super": 1543, "ultra": 1709 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 12.151, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-expense-report-1", "skill": "expense-report", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_ff34883f799d", "verdict_id": "verdict_ff34883f799d", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 22 replays. Review before merging. The system replayed 22 test cases against the skill change, which only updates the description from \"PDF or image files\" to \"PDF files only\" in SKILL.md. Twenty cases passed, but two were marked uncertain. In the first, the task asks to file ink and fuel receipts and email finance; the judge could not confirm if the description change affects this flow. In the second, the task is to clean signups.csv, which uses a different skill entirely, making the relevance unclear. No regressions, drifts, or improvements were found. Cost was minimal across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_60c12e465a52", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1129, "completion_tokens": 138, "usd": 0.00010086 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1601, "completion_tokens": 158, "usd": 0.0006225 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_87b3a1580656", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 188, "usd": 0.00012828 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_ac01bb0f765a", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only changes the description text from 'PDF or image files' to 'PDF files only' — a documentation-only change that does not alter any tool calls, arguments, or logic. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1602, "completion_tokens": 86, "usd": 0.00011676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3782, "completion_tokens": 419, "usd": 0.0015117 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3731, "completion_tokens": 1143, "usd": 0.00716 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_ba09cf46ac3d", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 198, "usd": 0.00012414 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_ca6090d7b4f3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1453, "completion_tokens": 137, "usd": 0.00012006 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_f88f40b0fdd4", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 220, "usd": 0.00013932 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_4c04fd872a46", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1303, "completion_tokens": 208, "usd": 0.0001281 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a5beee38284e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 105, "usd": 0.00010368 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37245d26f903", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1509, "completion_tokens": 172, "usd": 0.00013182 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_4e2b952bb502", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1447, "completion_tokens": 143, "usd": 0.00012114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_803282d49f8b", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 160, "usd": 0.00011238 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_c5f95c50e3d5", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1223, "completion_tokens": 197, "usd": 0.00012066 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_8a5a0ea0ed7f", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 181, "usd": 0.00011904 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2526, "completion_tokens": 301, "usd": 0.0010287 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_812811600cab", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 141, "usd": 0.00010782 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_eb78878a2ed4", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1269, "completion_tokens": 217, "usd": 0.00012822 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2855, "completion_tokens": 278, "usd": 0.0011067 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0932e89999f7", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 181, "usd": 0.00011904 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2684, "completion_tokens": 318, "usd": 0.0010914 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_154346b1b35e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 212, "usd": 0.00012726 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_388513c72ff0", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1361, "completion_tokens": 190, "usd": 0.00012726 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2864, "completion_tokens": 308, "usd": 0.0011364 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_1d8ff02cb8d4", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md (changing 'PDF or image files' to 'PDF files only'), which is unrelated to the CSV cleaning skill used for this task. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 187, "usd": 0.00012054 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2603, "completion_tokens": 297, "usd": 0.0010482 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2551, "completion_tokens": 1912, "usd": 0.008287 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_43b809e3abdd", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 109, "usd": 0.00009588 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1524, "completion_tokens": 212, "usd": 0.000648 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_62160dffa87e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 179, "usd": 0.00011316 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f2e5587a73ca", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 111, "usd": 0.00009792 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1567, "completion_tokens": 220, "usd": 0.0006681 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00260334, "super": 0.0088617, "ultra": 0.015447 }, "counterfactual_ultra_usd": 0.084715, "spent_usd_by_tier": { "nano": 0.00260334, "super": 0.0088617, "ultra": 0.016277 }, "prompt_tokens_by_tier": { "nano": 28749, "super": 22006, "ultra": 6656 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.662, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-2", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_7630dd45bd57", "verdict_id": "verdict_7630dd45bd57", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 7 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 7 of 13 replays. The sandbox caught 7 regressions where the edited skill made new github.comment calls that the original runs never made, spanning acme/infra-terraform, acme/auth-service, acme/data-pipeline, acme/docs-site, acme/mobile-app, acme/billing-api, and acme/widgets. Six replays passed unchanged. The change was reverted automatically. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_aeb36d32b26b", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 134, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_fb3621bc1546", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 137, "usd": 0.00010518 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_a026774bb5dc", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1348, "completion_tokens": 161, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_082c7d58917e", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_e82538fe8409", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_4281d13e3b01", "task_id": "tri-data-pipeline", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7ee8c01d51f9", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 1 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4f8712b7b45d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_650cb16fa7da", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_6f78304e2ae4", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_96b50023c3c8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1150, "completion_tokens": 114, "usd": 0.00009636 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_575a1d924d31", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 141, "usd": 0.00010404 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8419e2ca5496", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 109, "usd": 0.0000981 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0006237, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.009599, "spent_usd_by_tier": { "nano": 0.0006237, "super": 0.0, "ultra": 0.000877 }, "prompt_tokens_by_tier": { "nano": 7211, "super": 0, "ultra": 568 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 4.118, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-2", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_8205c47a1f05", "verdict_id": "verdict_8205c47a1f05", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 13 replays. Review before merging. The system re-ran 13 test pairs against the csv-cleaner skill change and found 10 passing with no regressions or drift. Three cases — cleaning payments.csv, cleaning signups.csv, and computing 17% of 240 — produced uncertain results where the skill diff did not explain the output differences, suggesting nondeterminism. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_959e7e0816aa", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 169, "usd": 0.00011478 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2239, "completion_tokens": 294, "usd": 0.0009363 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_b81786a373a5", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 150, "usd": 0.00011106 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_1dca9722e993", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 161, "usd": 0.00011118 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_96dcc8778ae9", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 164, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_130fa3316eae", "task_id": "csv-payments", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 101, "usd": 0.00010056 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2722, "completion_tokens": 295, "usd": 0.0010821 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2622, "completion_tokens": 1842, "usd": 0.008148 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_b7dba255cdef", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 221, "usd": 0.00012894 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_fe6d06e8fe47", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 208, "usd": 0.00012402 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_5d23e89670d4", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 151, "usd": 0.00011046 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2606, "completion_tokens": 335, "usd": 0.0010833 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2521, "completion_tokens": 3360, "usd": 0.012601 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_02fdaa1af68e", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 4505, "completion_tokens": 900, "usd": 0.00048630000000000006 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3333, "completion_tokens": 265, "usd": 0.0012384 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7c626553532c", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 151, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8b65fb6e74ff", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 105, "usd": 0.0000942 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1493, "completion_tokens": 223, "usd": 0.0006486 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f33745e12a8e", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 123, "usd": 0.000099 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1569, "completion_tokens": 219, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1445, "completion_tokens": 1410, "usd": 0.005675 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_690661728b24", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 119, "usd": 0.00009834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00180546, "super": 0.0056565, "ultra": 0.026424 }, "counterfactual_ultra_usd": 0.072647, "spent_usd_by_tier": { "nano": 0.00180546, "super": 0.0056565, "ultra": 0.027131 }, "prompt_tokens_by_tier": { "nano": 19199, "super": 13962, "ultra": 6956 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 17.661, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-2", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_ab3e8bf6820f", "verdict_id": "verdict_ab3e8bf6820f", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 13 replays. Review before merging. The skill diff adds rules for closing GitHub issues, but the replay pairs only test labeling behavior. One triage replay met all labeling criteria and used the correct tool calls yet was still held. A second triage replay was held because the new closing rules were irrelevant to the task. A math replay showed a calculation difference likely caused by nondeterminism rather than the skill change. Ten replays passed with no regressions or drift detected. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5da690453eaf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 99, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_66be25a1b831", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1206, "completion_tokens": 97, "usd": 0.00009564 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2427, "completion_tokens": 207, "usd": 0.0009144 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_9803f74fad28", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only adds rules about closing issues, which are irrelevant to the labeling task since no issues were requested to be closed. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1406, "completion_tokens": 94, "usd": 0.00010692 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3201, "completion_tokens": 336, "usd": 0.0012627 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3126, "completion_tokens": 2052, "usd": 0.009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_287402e48fdd", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1337, "completion_tokens": 114, "usd": 0.00010758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ad012c68b2ce", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1290, "completion_tokens": 141, "usd": 0.00011124 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_c31ee7fd6635", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets criteria 1-5: it states 4 open issues (m003), lists each with identifier and type (m003), mentions #406 already labeled (m003), states no details requested (m003), and uses github.label tool calls (t002-t004) indicating art…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1402, "completion_tokens": 131, "usd": 0.00011556 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3138, "completion_tokens": 299, "usd": 0.0012105 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3032, "completion_tokens": 3349, "usd": 0.013079 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_a3e558951509", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1430, "completion_tokens": 175, "usd": 0.0001278 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3010, "completion_tokens": 310, "usd": 0.001182 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7af9bb18724a", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1345, "completion_tokens": 99, "usd": 0.00010446 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9ea5f7e643be", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 177, "usd": 0.00012588 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_acc745f38e82", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 111, "usd": 0.00011136 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3ac050839ba", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 109, "usd": 0.00009444 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1521, "completion_tokens": 209, "usd": 0.0006444 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1e223b9eccf6", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 135, "usd": 0.0001026 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1573, "completion_tokens": 223, "usd": 0.0006726 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1507, "completion_tokens": 852, "usd": 0.004063 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a03ddffaa752", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 121, "usd": 0.0001017 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.001398, "super": 0.0058866, "ultra": 0.026424 }, "counterfactual_ultra_usd": 0.067743, "spent_usd_by_tier": { "nano": 0.001398, "super": 0.0058866, "ultra": 0.027225 }, "prompt_tokens_by_tier": { "nano": 16888, "super": 14870, "ultra": 8109 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.205, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_a1079119a111", "verdict_id": "verdict_a1079119a111", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The system replayed 13 test pairs against the skill change; 11 passed, but two were marked uncertain. In \"Convert at a weekend date,\" the replay used 2026-10-09 instead of the requested 2026-10-11. In \"Compute 17% of 240,\" the output difference was unexplained by the diff and likely stems from nondeterminism. Both cases were held for human review. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e5b902ded754", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 110, "usd": 0.00010512 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_8c4cb14a441e", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output shows the rate date as 2026-10-09 instead of the requested 2026-10-11, violating the 'uses the correct date 2026-10-11' criterion. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 151, "usd": 0.00011664 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2627, "completion_tokens": 306, "usd": 0.0010635 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2644, "completion_tokens": 1734, "usd": 0.007846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_3f28b051e158", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1328, "completion_tokens": 143, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_bdb089b8bdbb", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 121, "usd": 0.0001098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_2393bfd5f040", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1393, "completion_tokens": 236, "usd": 0.00014022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a029bb9d7e18", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1371, "completion_tokens": 203, "usd": 0.00013098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_148da7df7bf5", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 179, "usd": 0.00013008 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_c24c16f3b5fa", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 170, "usd": 0.00012066 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_b4e1ea78a932", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 118, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_cd2b69e99407", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 133, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_83397ea2c263", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 98, "usd": 0.0000918 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1586, "completion_tokens": 220, "usd": 0.0006738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_37f7a8b8bf4c", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1614, "completion_tokens": 229, "usd": 0.0006903 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1520, "completion_tokens": 1389, "usd": 0.005687 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8108d03dbe59", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 130, "usd": 0.00010176 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00147918, "super": 0.0024276, "ultra": 0.013533 }, "counterfactual_ultra_usd": 0.044414, "spent_usd_by_tier": { "nano": 0.00147918, "super": 0.0024276, "ultra": 0.014305 }, "prompt_tokens_by_tier": { "nano": 17197, "super": 5827, "ultra": 4540 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 10.671, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-1", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_dd861804440b", "verdict_id": "verdict_dd861804440b", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 13 replays. Review before merging. The system replayed the github-triage skill against 13 test cases after a one-line change to SKILL.md. Ten cases passed, but three were marked uncertain and held for human review. In \"Triage acme/data-pipeline,\" the replay appeared to meet all criteria but was flagged unsure at step t001. In \"Triage acme/billing-api\" and \"Compute 17% of 240,\" the judges cited likely nondeterminism because the skill diff did not explain the differences. No regressions, drift, or improvements were detected. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_bd63d6f067f1", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 139, "usd": 0.00010026 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f6019126d256", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1181, "completion_tokens": 144, "usd": 0.00010542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_dd87261b8ba6", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 138, "usd": 0.00011616 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_2d58c0602c67", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1368, "completion_tokens": 163, "usd": 0.0001212 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c9de1093aa99", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1307, "completion_tokens": 104, "usd": 0.00010338 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5101ae1012ad", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets criteria 1-5: final output states 4 open issues (m004), lists each with identifier and type (m004), mentions #406 already labeled (m004), states no details requested (m004), and trace shows github.label artifacts changed (t…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 133, "usd": 0.00011634 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3404, "completion_tokens": 294, "usd": 0.0012858 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3312, "completion_tokens": 3774, "usd": 0.014634 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_47ce9e153501", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1393, "completion_tokens": 203, "usd": 0.0001323 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3271, "completion_tokens": 308, "usd": 0.0012585 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4db894881c54", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1322, "completion_tokens": 116, "usd": 0.00010716 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_c8c4828aae44", "task_id": "tri-billing-api", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1409, "completion_tokens": 153, "usd": 0.00012126 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3536, "completion_tokens": 339, "usd": 0.0013659 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3477, "completion_tokens": 1515, "usd": 0.008022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_b3c44e403f10", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 143, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f3095a10d7d9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 104, "usd": 0.00009396 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1613, "completion_tokens": 223, "usd": 0.0006846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_4519fd683ef9", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1629, "completion_tokens": 219, "usd": 0.0006858 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1523, "completion_tokens": 1538, "usd": 0.006137 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_72d1ef164d37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 119, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014223, "super": 0.0052806, "ultra": 0.028793 }, "counterfactual_ultra_usd": 0.068369, "spent_usd_by_tier": { "nano": 0.0014223, "super": 0.0052806, "ultra": 0.029695 }, "prompt_tokens_by_tier": { "nano": 16781, "super": 13453, "ultra": 8737 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 18.091, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-1", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_d7659d3c93cf", "verdict_id": "verdict_d7659d3c93cf", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "judges_disagreed", "uncertain" ], "headline": "Held: judges disagreed on 1 of 13 replays. Review before merging.", "narrative": "Held: judges disagreed on 1 of 13 replays. Review before merging. The github-triage skill was edited with one change to SKILL.md. Automated evaluation ran 13 replay pairs: 12 passed, 1 was uncertain. The uncertain case, \"Compute 17% of 240,\" split the judges: Super flagged a regression while Ultra did not. Because the judges disagreed, the system held the change for human review. Cost was minimal across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_36523cbf4c92", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1127, "completion_tokens": 99, "usd": 0.00009138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e70cb39d820f", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 161, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_93bcdbcd3914", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 161, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ba25466de412", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1385, "completion_tokens": 183, "usd": 0.00012702 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_eb462443ad9b", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1295, "completion_tokens": 105, "usd": 0.0001029 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5d5629ab1937", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1405, "completion_tokens": 148, "usd": 0.00011982 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3092, "completion_tokens": 337, "usd": 0.0012309 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_37a737ab09a1", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 186, "usd": 0.000129 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2971, "completion_tokens": 294, "usd": 0.0011559 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a8f42769e490", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1329, "completion_tokens": 155, "usd": 0.00011694 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dbbba87c46a3", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1401, "completion_tokens": 157, "usd": 0.00012174 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_84d120ce3156", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 127, "usd": 0.00011448 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f1c02a8e442b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_227ef3c2f411", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "Judges disagreed: Super found a regression and Ultra did not. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1560, "completion_tokens": 213, "usd": 0.0006597 }, { "tier": "ultra", "verdict": "improved", "confidence": 0.95, "prompt_tokens": 1420, "completion_tokens": 949, "usd": 0.004267 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_ed4fe7d3e21d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 118, "usd": 0.00009954 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00143298, "super": 0.0030465, "ultra": 0.004267 }, "counterfactual_ultra_usd": 0.036535, "spent_usd_by_tier": { "nano": 0.00143298, "super": 0.0030465, "ultra": 0.004896 }, "prompt_tokens_by_tier": { "nano": 16803, "super": 7623, "ultra": 1710 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 11.099, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-2", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_8eac7d89877d", "verdict_id": "verdict_8eac7d89877d", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 1 of 22 past tasks.", "narrative": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 1 of 22 past tasks. The regression on \"File a bakery receipt and a missing one\" shows the skill no longer extracts required fields at step t004. Three other tasks were held for human review: an email send was blocked by the environment on \"File ink and fuel receipts and email finance,\" while \"Clean signups.csv\" and \"Compute 17% of 240\" were flagged as unrelated to the expense-report wording changes. Eighteen tasks passed. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_efb6b4383a27", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1149, "completion_tokens": 136, "usd": 0.00010158 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1862, "completion_tokens": 154, "usd": 0.0006972 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_26a650f173a0", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 171, "usd": 0.00012678 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3412, "completion_tokens": 299, "usd": 0.0012927 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3433, "completion_tokens": 3593, "usd": 0.014212 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_90c84e6db9bc", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay's gmail.send was blocked by the environment (t004: ratchet_blocked), preventing email delivery. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1553, "completion_tokens": 143, "usd": 0.0001275 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3978, "completion_tokens": 417, "usd": 0.0015687 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3923, "completion_tokens": 2361, "usd": 0.011006 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_9f99d91e21f6", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1301, "completion_tokens": 177, "usd": 0.00012054 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_29cce0ac5de3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1441, "completion_tokens": 141, "usd": 0.0001203 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_21bf7c25b237", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 184, "usd": 0.00013068 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_256b30ee0efe", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1303, "completion_tokens": 182, "usd": 0.00012186 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_dfea8f3d0de4", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1272, "completion_tokens": 143, "usd": 0.00011064 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_f1ddb4f3653c", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1497, "completion_tokens": 123, "usd": 0.00011934 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_ef0acc9e5a05", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 168, "usd": 0.00012786 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_11286bfe394d", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1185, "completion_tokens": 135, "usd": 0.0001035 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_325c7d095b64", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1256, "completion_tokens": 163, "usd": 0.00011448 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3011, "completion_tokens": 288, "usd": 0.0011625 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_2a529632a34a", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 142, "usd": 0.00010878 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_908da659aa18", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1203, "completion_tokens": 199, "usd": 0.00011994 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3a7988db6226", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 185, "usd": 0.00012144 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_3186365c17c2", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 197, "usd": 0.0001236 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2958, "completion_tokens": 302, "usd": 0.0011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_780ead4737e0", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 153, "usd": 0.00011094 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2457, "completion_tokens": 265, "usd": 0.0009756 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_0ba4c66e04ad", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 191, "usd": 0.0001239 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_58c7d9abe244", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies only the expense-report skill (wording changes like 'when'→'whenever', 'Steps'→'Procedure'), which is unrelated to the CSV cleaning task. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 105, "usd": 0.00009942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2812, "completion_tokens": 310, "usd": 0.0011226 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2749, "completion_tokens": 2564, "usd": 0.010441 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8e9bcc22b033", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 99, "usd": 0.00009204 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1746, "completion_tokens": 217, "usd": 0.0007191 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1cc2a5f894e5", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies an expense-report skill unrelated to the math task 'What is 17% of 240?'. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1798, "completion_tokens": 219, "usd": 0.0007365 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1710, "completion_tokens": 1438, "usd": 0.006024 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_07aa4d542d5a", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 102, "usd": 0.00009642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0025083, "super": 0.0094341, "ultra": 0.041683 }, "counterfactual_ultra_usd": 0.111624, "spent_usd_by_tier": { "nano": 0.0025083, "super": 0.0094341, "ultra": 0.042558 }, "prompt_tokens_by_tier": { "nano": 28561, "super": 24034, "ultra": 12285 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.627, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_6e2d8e72750d", "verdict_id": "verdict_6e2d8e72750d", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs against the updated fx-expense-converter skill and found 12 passing with no regressions, drift, or improvements. One case, \"Compute 17% of 240,\" produced an uncertain result because the skill change does not explain the output difference, suggesting possible nondeterminism. The run cost fractions of a cent across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_8c2c5e26d4e1", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 129, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_2a31e3e43325", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1341, "completion_tokens": 207, "usd": 0.00013014 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_40498db1a5f2", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 159, "usd": 0.00011712 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_0f0b98c0dda1", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1536, "completion_tokens": 122, "usd": 0.00012144 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3048, "completion_tokens": 330, "usd": 0.0012114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_a341fb0948fa", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 243, "usd": 0.00014046 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_b5a453299350", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1383, "completion_tokens": 237, "usd": 0.00013986 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ea5e1042bbf8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1427, "completion_tokens": 113, "usd": 0.00011274 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ab59cb62584a", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 164, "usd": 0.0001185 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_d101d55224e4", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1416, "completion_tokens": 123, "usd": 0.00011448 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_f96f5f3cef70", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1388, "completion_tokens": 132, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_eb8645af7d9f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 102, "usd": 0.00009492 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1645, "completion_tokens": 223, "usd": 0.0006942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_710563b99aef", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1685, "completion_tokens": 224, "usd": 0.0007071 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1566, "completion_tokens": 1829, "usd": 0.007053 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb5c11b812f5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 111, "usd": 0.00009792 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014997, "super": 0.0026127, "ultra": 0.007053 }, "counterfactual_ultra_usd": 0.038843, "spent_usd_by_tier": { "nano": 0.0014997, "super": 0.0026127, "ultra": 0.007694 }, "prompt_tokens_by_tier": { "nano": 17339, "super": 6378, "ultra": 1865 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 12.997, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_1c337d2ebbfb", "verdict_id": "verdict_1c337d2ebbfb", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The skill diff only adds a condition to omit the rate line when the rate equals 1.0000. Eleven test pairs passed with no regressions or drift. Two pairs were held for human review. In \"Convert at a weekend date,\" the rate is 1.0768 so the new condition does not apply, yet the judge was uncertain. In \"Compute 17% of 240,\" the output contains the expected numeric result and phrasing, but the judge was not sure. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_88832e95c9b9", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 129, "usd": 0.00010968 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_33a9c6adf157", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a condition to omit the rate line when rate=1.0000, which does not apply here (rate=1.0768). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 104, "usd": 0.00010392 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2685, "completion_tokens": 297, "usd": 0.0010728 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2612, "completion_tokens": 1843, "usd": 0.008141 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_07551b4c2872", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 247, "usd": 0.0001368 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f725843c1f80", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1394, "completion_tokens": 121, "usd": 0.00011268 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_0545f282df0a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 192, "usd": 0.00012894 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9df69746323e", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1384, "completion_tokens": 162, "usd": 0.00012192 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_51bbbd84f8be", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 136, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_32ea0f012a38", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1343, "completion_tokens": 166, "usd": 0.00012042 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_1847ee62554a", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 119, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_08c9424f7da6", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 189, "usd": 0.00013008 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_a144e4814df4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 109, "usd": 0.00009444 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1621, "completion_tokens": 223, "usd": 0.000687 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e625a12fe4a6", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay output '17% of 240 = 0.17 × 240 = 40.8.' contains the numeric result 40.8, the phrase '17% of 240', the phrase '= 0.17 × 240', and the value 40.8 after an equals sign. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1673, "completion_tokens": 226, "usd": 0.0007053 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1550, "completion_tokens": 1161, "usd": 0.005033 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c818c2acd85b", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 108, "usd": 0.0000972 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1664, "completion_tokens": 217, "usd": 0.0006945 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00147684, "super": 0.0031596, "ultra": 0.013174 }, "counterfactual_ultra_usd": 0.046466, "spent_usd_by_tier": { "nano": 0.00147684, "super": 0.0031596, "ultra": 0.014028 }, "prompt_tokens_by_tier": { "nano": 17198, "super": 7643, "ultra": 4584 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 9.924, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_1bfebab75717", "verdict_id": "verdict_1bfebab75717", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs against the updated fx-expense-converter skill and found 12 passing with no regressions or drift. One case, \"Compute 17% of 240,\" produced an uncertain result because the skill change does not explain the output difference, suggesting possible nondeterminism. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e9f830fa642e", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1346, "completion_tokens": 189, "usd": 0.00012612 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_bbae78eada7e", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1350, "completion_tokens": 126, "usd": 0.00011124 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_518252b4621f", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1326, "completion_tokens": 209, "usd": 0.00012972 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2718, "completion_tokens": 293, "usd": 0.0010791 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_6bd370a9ae4d", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1392, "completion_tokens": 88, "usd": 0.00010464 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_4a19b9d5037d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1391, "completion_tokens": 230, "usd": 0.00013866 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.98, "prompt_tokens": 2718, "completion_tokens": 339, "usd": 0.0011205 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_659d9e0a0119", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 207, "usd": 0.00013404 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ae6602b84c3d", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1435, "completion_tokens": 142, "usd": 0.00012018 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3047, "completion_tokens": 374, "usd": 0.0012507 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_76bd7f61f20c", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1377, "completion_tokens": 220, "usd": 0.00013542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_7b5ee5b9fe63", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1402, "completion_tokens": 119, "usd": 0.00011268 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_40cfb64bd6bb", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1420, "completion_tokens": 133, "usd": 0.00011712 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3097, "completion_tokens": 390, "usd": 0.0012801 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2c596c557862", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 101, "usd": 0.00009252 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1662, "completion_tokens": 221, "usd": 0.0006975 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_dd37987a313e", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1678, "completion_tokens": 224, "usd": 0.000705 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1595, "completion_tokens": 1263, "usd": 0.005384 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_14951b257b04", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 82, "usd": 0.00008946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00149784, "super": 0.0061329, "ultra": 0.005384 }, "counterfactual_ultra_usd": 0.048873, "spent_usd_by_tier": { "nano": 0.00149784, "super": 0.0061329, "ultra": 0.00598 }, "prompt_tokens_by_tier": { "nano": 17292, "super": 14920, "ultra": 1894 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 10.302, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-2", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_bf5dc4bb6742", "verdict_id": "verdict_bf5dc4bb6742", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now omits the repository name in the format `acme/` on 1 of 13 past tasks.", "narrative": "Blocked: github-triage now omits the repository name in the format `acme/` on 1 of 13 past tasks. The evaluation ran 13 regression pairs against the skill edit; 10 passed, 1 regressed, and 2 were uncertain. The regression shows the agent no longer states the repository name in the required format during triage. One uncertain case involves a triage task held for human review because the replay met criteria but was flagged. The other uncertain case is a math calculation where the skill diff does not explain the difference, suggesting nondeterminism. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0e10ad5bac34", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the repository name in the format `acme/` is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1559, "completion_tokens": 241, "usd": 0.0006846 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1544, "completion_tokens": 1852, "usd": 0.0071 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e27ff37067f7", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1217, "completion_tokens": 175, "usd": 0.00011502 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3d9dd364ab27", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 179, "usd": 0.00012456 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_570420a9b31d", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 137, "usd": 0.00011526 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_40e6b8dc80bd", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1304, "completion_tokens": 166, "usd": 0.00011808 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3040, "completion_tokens": 279, "usd": 0.0011631 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5a38101f8f14", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets criteria 1-4: it states 4 open issues, lists each with identifier and type, mentions #406 already labeled, and states no details were requested. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 102, "usd": 0.00010944 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3161, "completion_tokens": 299, "usd": 0.0012174 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3025, "completion_tokens": 2516, "usd": 0.010573 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_1f166314792c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 243, "usd": 0.0001437 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3012, "completion_tokens": 296, "usd": 0.00117 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d9f90397db42", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1305, "completion_tokens": 137, "usd": 0.00011118 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_13d086a4d0a2", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1385, "completion_tokens": 163, "usd": 0.00012222 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_24cc0a76ffb7", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1380, "completion_tokens": 124, "usd": 0.00011256 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_86211c1cba50", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 116, "usd": 0.00009756 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_70087be29148", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 64, "usd": 0.00008412 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1559, "completion_tokens": 219, "usd": 0.0006648 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1426, "completion_tokens": 1555, "usd": 0.006091 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3b107f1aecf5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 110, "usd": 0.00009762 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00144456, "super": 0.0048999, "ultra": 0.023764 }, "counterfactual_ultra_usd": 0.062359, "spent_usd_by_tier": { "nano": 0.00144456, "super": 0.0048999, "ultra": 0.024558 }, "prompt_tokens_by_tier": { "nano": 16820, "super": 12331, "ultra": 6396 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.48, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_6e96238790c2", "verdict_id": "verdict_6e96238790c2", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The skill diff only adds a comment line and changes no executable logic. Eleven replays passed, but two were held for human review. The \"Convert at a weekend date\" replay was flagged because the diff does not explain the behavioral difference. The \"Compute 17% of 240\" replay was flagged as likely nondeterminism. No regressions or drifts were detected. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_4e739b0fed3d", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1324, "completion_tokens": 135, "usd": 0.00011184 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_d7ede738861c", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a comment line and changes no step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 151, "usd": 0.00011664 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2569, "completion_tokens": 283, "usd": 0.0010254 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2528, "completion_tokens": 2558, "usd": 0.010202 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_7f573055a9ef", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 168, "usd": 0.00011928 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f4c1c98c1a21", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 92, "usd": 0.000105 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_6b7462d13be4", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 227, "usd": 0.00013734 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_914ca7824ded", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1384, "completion_tokens": 152, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_da9dc2ce34c7", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1416, "completion_tokens": 135, "usd": 0.00011736 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_70ade45e8144", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1343, "completion_tokens": 219, "usd": 0.00013314 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_0b662be246d7", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 118, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_ca8b25ae83a2", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 139, "usd": 0.00011736 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_0bda149accdc", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 97, "usd": 0.00009228 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1482, "completion_tokens": 211, "usd": 0.0006345 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e0d76dcf56dd", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1522, "completion_tokens": 219, "usd": 0.0006537 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1404, "completion_tokens": 1484, "usd": 0.005856 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f3b045e8f40e", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 102, "usd": 0.0000957 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00146694, "super": 0.0023136, "ultra": 0.016058 }, "counterfactual_ultra_usd": 0.046412, "spent_usd_by_tier": { "nano": 0.00146694, "super": 0.0023136, "ultra": 0.016755 }, "prompt_tokens_by_tier": { "nano": 17221, "super": 5573, "ultra": 4272 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.08, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-2", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_12058884cc20", "verdict_id": "verdict_12058884cc20", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "level0_hard_fail" ], "headline": "Blocked: the new github-triage would comment on issues in 1 of 13 replays.", "narrative": "Blocked: the new github-triage would comment on issues in 1 of 13 replays. The system replayed 13 test cases against the proposed change; 10 passed, 1 regressed, and 2 were uncertain. The regression occurred in \"Triage acme/infra-terraform\" where the new code attempted two github.comment calls the original never made, which the sandbox blocked. One uncertain case, \"Triage acme/data-pipeline,\" produced only three github.label calls instead of the required six. The other uncertain case, a math task, showed unexplained variance likely due to nondeterminism. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_9aae4e6186d2", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 99, "usd": 0.00009282 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_852cf7299d83", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1181, "completion_tokens": 164, "usd": 0.00011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_1f4cd7269ecb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 157, "usd": 0.00012072 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d6db457362fe", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "level0", "reason": "Checked without a model: 2 new write_irreversible call(s).", "opinions": [], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_7d366fbfea40", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 105, "usd": 0.00010434 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_bb40249ba200", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets criteria 1-5 but fails criterion 6 (only 3 github.label calls vs required 6). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "uncertain", "confidence": 0.0, "prompt_tokens": 4868, "completion_tokens": 900, "usd": 0.00050808 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3143, "completion_tokens": 282, "usd": 0.0011967 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2971, "completion_tokens": 3238, "usd": 0.012685 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_5cb5a86b5b0c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 199, "usd": 0.00013062 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3148, "completion_tokens": 271, "usd": 0.0011883 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_8e097e2bc153", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1341, "completion_tokens": 114, "usd": 0.00010782 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dc72a3a720da", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1366, "completion_tokens": 189, "usd": 0.00012732 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d4e29e63139d", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1392, "completion_tokens": 145, "usd": 0.00011832 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3365, "completion_tokens": 302, "usd": 0.0012813 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1bcf329ed33b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_20de0884c5fb", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1558, "completion_tokens": 219, "usd": 0.0006645 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1446, "completion_tokens": 1455, "usd": 0.005811 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b123eb9839bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 146, "usd": 0.00010698 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00170724, "super": 0.0043308, "ultra": 0.018496 }, "counterfactual_ultra_usd": 0.058998, "spent_usd_by_tier": { "nano": 0.00170724, "super": 0.0043308, "ultra": 0.019295 }, "prompt_tokens_by_tier": { "nano": 18902, "super": 11214, "ultra": 4808 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 24.294, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-1", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_63815fa9f45c", "verdict_id": "verdict_63815fa9f45c", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now fails \"does not mention any other issue numbers or open issues beyond the two listed\" on 1 of 13 past tasks.", "narrative": "Blocked: github-triage now fails \"does not mention any other issue numbers or open issues beyond the two listed\" on 1 of 13 past tasks. The edit only reordered two documentation lines in SKILL.md. Regression testing caught a failure on the acme/status-page task where the agent now mentions extra issues. Three other tasks were held for human review due to uncertainty, including a nondeterministic math task and two triage tasks where criteria appeared met but required verification. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_fc1895f58dcf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1127, "completion_tokens": 99, "usd": 0.00009138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_99daafbd39ef", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "does not mention any other issue numbers or open issues beyond the two listed is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1240, "completion_tokens": 174, "usd": 0.00011616 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2658, "completion_tokens": 272, "usd": 0.0010422 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2649, "completion_tokens": 4304, "usd": 0.015561 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_204e00dc72de", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only reorders two documentation lines in SKILL.md (swapping 'Already labeled' and 'Asked for details' bullet points) without changing any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1341, "completion_tokens": 101, "usd": 0.0001047 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3296, "completion_tokens": 293, "usd": 0.0012525 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3220, "completion_tokens": 4135, "usd": 0.015625 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d8383ef71ebb", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1368, "completion_tokens": 135, "usd": 0.00011448 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3305, "completion_tokens": 262, "usd": 0.0012273 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_f644709d32f1", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1304, "completion_tokens": 142, "usd": 0.00011232 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3046, "completion_tokens": 249, "usd": 0.0011379 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_92e4b2c50b1c", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1383, "completion_tokens": 95, "usd": 0.00010578 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3356, "completion_tokens": 384, "usd": 0.0013524 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_574e79e6ac2e", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1404, "completion_tokens": 175, "usd": 0.00012624 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3033, "completion_tokens": 314, "usd": 0.0011925 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7213bfe2b9f8", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "All six success criteria are met by the replay run: it states 4 open issues (m004), labels #58 bug (t002), #61 enhancement (t003), #64 question (t004), acknowledges #62 as already labeled (m004), and asks for details on #64 via github.comme…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1331, "completion_tokens": 133, "usd": 0.00011178 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3072, "completion_tokens": 326, "usd": 0.001215 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 6580, "completion_tokens": 6154, "usd": 0.025042 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_a05f5333a824", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1401, "completion_tokens": 177, "usd": 0.00012654 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 3677, "completion_tokens": 292, "usd": 0.0013659 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_1891e80878db", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1400, "completion_tokens": 155, "usd": 0.0001212 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3218, "completion_tokens": 294, "usd": 0.00123 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d5662e056952", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 100, "usd": 0.00009228 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1517, "completion_tokens": 221, "usd": 0.000654 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e3d06ab590fa", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 115, "usd": 0.00009708 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1605, "completion_tokens": 214, "usd": 0.0006741 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1526, "completion_tokens": 1685, "usd": 0.006581 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_18d4cf624d2d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 100, "usd": 0.00009378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00141372, "super": 0.0123438, "ultra": 0.062809 }, "counterfactual_ultra_usd": 0.125816, "spent_usd_by_tier": { "nano": 0.00141372, "super": 0.0123438, "ultra": 0.063638 }, "prompt_tokens_by_tier": { "nano": 16758, "super": 31783, "ultra": 14489 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 33.961, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-expense-report-2", "skill": "expense-report", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_335957a29e63", "verdict_id": "verdict_335957a29e63", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 1 of 22 past tasks.", "narrative": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 1 of 22 past tasks. The system ran 22 regression pairs and found 16 passing, 1 regression, and 5 uncertain. The regression is a clear failure on a bakery receipt task where required fields disappeared. The uncertain cases involve external factors like Gmail blocking, CSV row count mismatches from unrelated data changes, and a skill diff mismatch on a payments cleaner task. One math task passed all criteria but was held for review. The change breaks a known working behavior and the uncertain results do not offset that loss. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2f14f21d6d9f", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1106, "completion_tokens": 157, "usd": 0.00010404 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_e92a7282295d", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1416, "completion_tokens": 142, "usd": 0.00011904 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3150, "completion_tokens": 306, "usd": 0.0012204 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3102, "completion_tokens": 3435, "usd": 0.013407 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_a6b930ea6aff", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails the email-sent criterion because gmail.send was blocked with 'ratchet_blocked' and the output states 'The email could not be sent due to system restrictions'. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1654, "completion_tokens": 124, "usd": 0.000129 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 4021, "completion_tokens": 373, "usd": 0.001542 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 4006, "completion_tokens": 1599, "usd": 0.008803 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_e2914d406490", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1289, "completion_tokens": 140, "usd": 0.00011094 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_07a85852522a", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 147, "usd": 0.00012102 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_85a67c3ff72c", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1454, "completion_tokens": 156, "usd": 0.00012468 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_afcfe0768c5a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1291, "completion_tokens": 192, "usd": 0.00012354 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a64afbf6b2cb", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1272, "completion_tokens": 148, "usd": 0.00011184 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_9c6ec62f2eb9", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 124, "usd": 0.00012102 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6ad0a889dbc3", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1471, "completion_tokens": 195, "usd": 0.00013506 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_b6e697e319df", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 164, "usd": 0.00011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_8ffffbf26485", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1256, "completion_tokens": 168, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_56a48c4930ba", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 147, "usd": 0.00010926 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_4c8e63150ce8", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 203, "usd": 0.00012432 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_aa8e42f0327e", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay produces 4 data rows instead of the required 3, violating the success criteria. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1373, "completion_tokens": 94, "usd": 0.00010494 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3375, "completion_tokens": 330, "usd": 0.0013095 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 7085, "completion_tokens": 6027, "usd": 0.025166 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_c3600f8c1037", "task_id": "csv-payments", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md but the golden and replay runs both execute a CSV cleaner on payments.csv, which is a different skill entirely. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 128, "usd": 0.00010632 }, { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 2841, "completion_tokens": 72, "usd": 0.0009171 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2752, "completion_tokens": 2246, "usd": 0.00949 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_64e31b08b933", "task_id": "csv-header-only", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The leads.csv file contained only a header in the golden run but had 7 data rows in the replay run (t001 vs t003), an external fact change unrelated to the skill diff. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 234, "usd": 0.0001335 }, { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 3010, "completion_tokens": 83, "usd": 0.0009777 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2976, "completion_tokens": 5108, "usd": 0.0183 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_c2e995113d65", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 183, "usd": 0.00012126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_18bb09c7d462", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1225, "completion_tokens": 216, "usd": 0.00012534 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_de7f8d3695ad", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 125, "usd": 0.00009828 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1620, "completion_tokens": 209, "usd": 0.0006741 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_40e8a755ae4f", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "All success criteria are met by the replay output: it contains the numeric result 40.8, the phrase '17% of 240', the phrase '= 0.17 × 240', and the value 40.8 after an equals sign. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 73, "usd": 0.00008772 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1672, "completion_tokens": 226, "usd": 0.000705 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1569, "completion_tokens": 1192, "usd": 0.005145 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b58e85552abd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 134, "usd": 0.0001041 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00254424, "super": 0.0073458, "ultra": 0.080311 }, "counterfactual_ultra_usd": 0.143807, "spent_usd_by_tier": { "nano": 0.00254424, "super": 0.0073458, "ultra": 0.08135 }, "prompt_tokens_by_tier": { "nano": 28828, "super": 19689, "ultra": 22106 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 54.521, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-expense-report-1", "skill": "expense-report", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_5a6139f9aec5", "verdict_id": "verdict_5a6139f9aec5", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 9 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 9 of 22 replays. Review before merging. The diff only rewords the skill description without changing logic, yet 9 replays were marked uncertain because outputs missed required totals or the skill change was unrelated to the task. Thirteen replays passed with no regressions. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2cfa611323e6", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1094, "completion_tokens": 128, "usd": 0.00009636 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_f3f61876a85b", "task_id": "exp-missing-receipt", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field (receipts → receipts, invoices, or other financial documents; receipt file → file; receipt → document) without changing any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1442, "completion_tokens": 142, "usd": 0.0001206 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 2804, "completion_tokens": 271, "usd": 0.0010851 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2756, "completion_tokens": 2041, "usd": 0.008879 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_03e985bf60c4", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description in SKILL.md (changing 'receipts' to 'documents') without altering any tool calls, arguments, or logic. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1589, "completion_tokens": 97, "usd": 0.00011862 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3825, "completion_tokens": 414, "usd": 0.0015201 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3743, "completion_tokens": 1859, "usd": 0.00932 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_b14c6d6a1857", "task_id": "exp-software", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output misses the 'Grand total: 117.81 EUR' line required by success criteria, while the golden output includes it. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1338, "completion_tokens": 175, "usd": 0.00012228 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2634, "completion_tokens": 289, "usd": 0.0010503 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2627, "completion_tokens": 1471, "usd": 0.00704 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_758cddadc695", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1513, "completion_tokens": 153, "usd": 0.0001275 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3346, "completion_tokens": 327, "usd": 0.0012981 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5e47aca6907b", "task_id": "exp-dinner-parking", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output meets 5 of 6 success criteria: it states merchant, date, total, and VAT for each receipt (m004), states both merchant names (m004), states the date 2026-10-07 for both (m004), states the total amounts 125.00 EUR and 19.00…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 1456, "completion_tokens": 112, "usd": 0.00011424 }, { "tier": "super", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3129, "completion_tokens": 303, "usd": 0.0012114 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3071, "completion_tokens": 2731, "usd": 0.011264 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a8a18a97157f", "task_id": "exp-office-supplies", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field in SKILL.md without changing any tool, rule, or logic. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1327, "completion_tokens": 121, "usd": 0.00010866 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2499, "completion_tokens": 274, "usd": 0.0009963 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2356, "completion_tokens": 1421, "usd": 0.006619 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_2a8fe0325d1e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1320, "completion_tokens": 143, "usd": 0.00011352 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_21460568387a", "task_id": "exp-lunch-train", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field (receipts → receipts, invoices, or other financial documents) without changing any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1517, "completion_tokens": 176, "usd": 0.00013326 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3309, "completion_tokens": 379, "usd": 0.0013338 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3316, "completion_tokens": 2982, "usd": 0.012262 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_1b2e69a62019", "task_id": "exp-berlin-trip", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field (receipts→documents) without changing any tool, rule, or step logic. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1441, "completion_tokens": 146, "usd": 0.0001215 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3163, "completion_tokens": 348, "usd": 0.0012621 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3124, "completion_tokens": 1752, "usd": 0.00838 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_df0530c17b4e", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 150, "usd": 0.0001107 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_5b34a19cd981", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 154, "usd": 0.00011106 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_b1ea18acb350", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 121, "usd": 0.00010374 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_6d6c20462d5a", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 203, "usd": 0.00012378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_1d5047ec8fd2", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The cleaned file in both runs contains exactly 3 data rows with unique non-empty IDs and no duplicates, meeting all success criteria (evidence: t003 tool results in both runs show identical correct output). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 101, "usd": 0.00010128 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2706, "completion_tokens": 280, "usd": 0.0010638 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2605, "completion_tokens": 3538, "usd": 0.013219 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_a7ca6fd2483c", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 136, "usd": 0.00010824 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2652, "completion_tokens": 300, "usd": 0.0010656 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_8ed25d9902cb", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1213, "completion_tokens": 178, "usd": 0.0001155 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2195, "completion_tokens": 298, "usd": 0.0009267 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_bdb9bd3422e8", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 181, "usd": 0.0001215 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_03c42fb9fb74", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff modifies expense-report/SKILL.md, which is unrelated to the CSV cleaner skill used for tidying signups.csv. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 128, "usd": 0.00010566 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2610, "completion_tokens": 292, "usd": 0.0010458 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2553, "completion_tokens": 2509, "usd": 0.01008 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d54e294c1ad8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 81, "usd": 0.00008916 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3cb515f6830e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 119, "usd": 0.00009876 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1629, "completion_tokens": 205, "usd": 0.0006732 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c51d156486d2", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 82, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00245682, "super": 0.0145323, "ultra": 0.087063 }, "counterfactual_ultra_usd": 0.173424, "spent_usd_by_tier": { "nano": 0.00245682, "super": 0.0145323, "ultra": 0.088104 }, "prompt_tokens_by_tier": { "nano": 28839, "super": 36501, "ultra": 26967 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 31.465, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-1", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_cbe6352873c4", "verdict_id": "verdict_cbe6352873c4", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 4 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 4 of 22 replays. Review before merging. The diff only adds a comment line to SKILL.md with no logic changes. Eighteen pairs passed, but four were held for human review due to missing file writes, unexplained output differences, and nondeterminism that the diff does not explain. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_9797515450fd", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1070, "completion_tokens": 103, "usd": 0.00008892 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_562f1bacbb4a", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1405, "completion_tokens": 116, "usd": 0.00011214 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3197, "completion_tokens": 269, "usd": 0.0012012 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_118b690af2c2", "task_id": "exp-ink-fuel-email", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a comment line to SKILL.md and changes no logic, tools, or arguments. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1670, "completion_tokens": 121, "usd": 0.00012924 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3822, "completion_tokens": 428, "usd": 0.0015318 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3790, "completion_tokens": 1524, "usd": 0.008362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_17c39cd8a28a", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1313, "completion_tokens": 188, "usd": 0.0001239 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_89a8beb01ad9", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 148, "usd": 0.00012126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5a08a4d190cc", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1466, "completion_tokens": 109, "usd": 0.00011412 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a04984da2f2e", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1315, "completion_tokens": 175, "usd": 0.0001209 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_6c284a0936b6", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1332, "completion_tokens": 146, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37dcfb667a2b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1533, "completion_tokens": 127, "usd": 0.00012246 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_f63fa53e365f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1471, "completion_tokens": 113, "usd": 0.00011538 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_596660827ddb", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 152, "usd": 0.00011118 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_35dd5c46b4f3", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1259, "completion_tokens": 154, "usd": 0.0001125 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_24bcb89d6737", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 148, "usd": 0.0001095 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_3053a2b19841", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 132, "usd": 0.00010602 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_df65d3ad0120", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 173, "usd": 0.00011784 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_175a9b556a3d", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 130, "usd": 0.00010536 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2640, "completion_tokens": 330, "usd": 0.001089 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_6396f7029b97", "task_id": "csv-header-only", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay run did not call fs.write to create leads.clean.csv (Level 0 note: missing fs.write), while the golden run did (t003), so criterion 5 fails in replay. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1358, "completion_tokens": 183, "usd": 0.0001254 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2464, "completion_tokens": 239, "usd": 0.0009543 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2405, "completion_tokens": 4197, "usd": 0.014996 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_7ff7643575cd", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 190, "usd": 0.00012294 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_07fbc9fb046d", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 122, "usd": 0.00010422 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2572, "completion_tokens": 303, "usd": 0.0010443 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2497, "completion_tokens": 1752, "usd": 0.007753 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_11e6e85db7b9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 121, "usd": 0.00009804 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1473, "completion_tokens": 223, "usd": 0.0006426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_6781c847f496", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a comment line to SKILL.md and does not change any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 69, "usd": 0.00008532 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1513, "completion_tokens": 219, "usd": 0.000651 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1409, "completion_tokens": 1087, "usd": 0.00467 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_16136c6d8859", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 111, "usd": 0.00009858 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00246018, "super": 0.0071142, "ultra": 0.035781 }, "counterfactual_ultra_usd": 0.097467, "spent_usd_by_tier": { "nano": 0.00246018, "super": 0.0071142, "ultra": 0.036501 }, "prompt_tokens_by_tier": { "nano": 28879, "super": 17681, "ultra": 10554 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 23.148, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-1", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_71b7ae8b235d", "verdict_id": "verdict_71b7ae8b235d", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 13 replays. Review before merging. The edit only added a cosmetic comment to SKILL.md with no logic changes. Ten replays passed, but three were held uncertain: two CSV cleaning tasks where the diff showed no relevant change, and one math task likely affected by nondeterminism. No regressions, drift, or improvements were detected. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_95b0aeaae315", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 196, "usd": 0.00012342 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ad0ef0f6f262", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1227, "completion_tokens": 152, "usd": 0.0001101 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_4bc22f69c2f4", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1257, "completion_tokens": 149, "usd": 0.00011118 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_341f1aafe06d", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a cosmetic comment to SKILL.md and does not change any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 143, "usd": 0.00010848 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2681, "completion_tokens": 280, "usd": 0.0010563 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2632, "completion_tokens": 2901, "usd": 0.011335 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_89cbeb5f1106", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 238, "usd": 0.00013272 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2609, "completion_tokens": 309, "usd": 0.0010608 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_638b93eff172", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 191, "usd": 0.00012318 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_57f1a1c928ec", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1247, "completion_tokens": 166, "usd": 0.00011466 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_ed700d3411da", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a comment line to SKILL.md and does not change any cleaning logic, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 175, "usd": 0.00011838 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2507, "completion_tokens": 298, "usd": 0.0010203 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2463, "completion_tokens": 4007, "usd": 0.014484 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_f83b9f46b05b", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 108, "usd": 0.00010218 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7323ba2c2619", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 128, "usd": 0.00010542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2e08956d5846", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 97, "usd": 0.00009372 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_8ad7f2f013aa", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1517, "completion_tokens": 219, "usd": 0.0006522 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 1137, "usd": 0.00481 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3f0ad4c092bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 106, "usd": 0.00009522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142542, "super": 0.0037896, "ultra": 0.030629 }, "counterfactual_ultra_usd": 0.065097, "spent_usd_by_tier": { "nano": 0.00142542, "super": 0.0037896, "ultra": 0.031319 }, "prompt_tokens_by_tier": { "nano": 16073, "super": 9314, "ultra": 6887 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 17.884, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-github-triage-1", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_40f4622e4642", "verdict_id": "verdict_40f4622e4642", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 13 replays. Review before merging. The system replayed 13 test cases against the updated github-triage skill; 10 passed, but 3 were held as uncertain. In \"Triage acme/data-pipeline\", the replay produced only three label changes versus six in the golden trace. In \"Triage acme/mobile-app\" and \"Compute 17% of 240\", the differences could not be explained by the skill change and appear to be nondeterministic. No regressions or drifts were detected. Cost was negligible across all model tiers. Inspect the three uncertain replays to determine if the new behavior is correct. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_c6a8e1abb734", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 121, "usd": 0.00009882 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_4e23436145b0", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 86, "usd": 0.00009294 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e1dd839734fb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 133, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_860252f0aeb1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1380, "completion_tokens": 137, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac0076e8ac87", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1307, "completion_tokens": 126, "usd": 0.00010866 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_0247b5972b80", "task_id": "tri-data-pipeline", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay violates the criterion 'specifies that exactly six github.label changes occurred' — only three github.label calls appear in the replay trace (t002, t003, t004) versus six in the golden trace (t002–t007). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1408, "completion_tokens": 162, "usd": 0.00012336 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3092, "completion_tokens": 316, "usd": 0.001212 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3003, "completion_tokens": 3041, "usd": 0.012126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_07bb9b326000", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1478, "completion_tokens": 217, "usd": 0.00014076 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3109, "completion_tokens": 358, "usd": 0.0012549 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_ef274337a616", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 129, "usd": 0.00011136 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2982, "completion_tokens": 307, "usd": 0.0011709 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2956, "completion_tokens": 2181, "usd": 0.009499 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_ee6719208401", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1409, "completion_tokens": 146, "usd": 0.00011958 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_44c02d5b3570", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1389, "completion_tokens": 143, "usd": 0.00011766 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3173, "completion_tokens": 348, "usd": 0.0012651 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_bfdc8aa6fc39", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 100, "usd": 0.00009228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a63316f74650", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 72, "usd": 0.00008532 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1515, "completion_tokens": 221, "usd": 0.0006534 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 1321, "usd": 0.005362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_86b52a6672e1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1175, "completion_tokens": 84, "usd": 0.00009066 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00141204, "super": 0.0055563, "ultra": 0.026987 }, "counterfactual_ultra_usd": 0.067386, "spent_usd_by_tier": { "nano": 0.00141204, "super": 0.0055563, "ultra": 0.027883 }, "prompt_tokens_by_tier": { "nano": 16910, "super": 13871, "ultra": 7777 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.933, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_e3bbcfdc38b1", "verdict_id": "verdict_e3bbcfdc38b1", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The system replayed 13 test pairs against the updated skill; 11 passed, but two were marked uncertain. The London cost conversion used an exchange rate of 1.3105 while the golden run expected 1.2702 for 2026-10-13, shifting all USD results. The percentage calculation also diverged without a clear cause in the skill diff, suggesting nondeterminism. Both cases were held for human review. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_3e8797c16149", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 135, "usd": 0.00011256 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_608d63d15c36", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 160, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_229dc213dbd6", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 168, "usd": 0.00011856 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_e71800e8113a", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1406, "completion_tokens": 102, "usd": 0.00010884 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_16ce4eaa1d1a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 268, "usd": 0.00014646 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_bd178338e4b1", "task_id": "fx-london", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The replay uses exchange rate 1.3105 while the golden run uses 1.2702 for 2026-10-13, causing all USD conversions to differ. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 143, "usd": 0.00011874 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2584, "completion_tokens": 305, "usd": 0.0010497 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2586, "completion_tokens": 1427, "usd": 0.006867 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_f6108436cd3f", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1451, "completion_tokens": 144, "usd": 0.00012162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_133f03cc5707", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 212, "usd": 0.00013218 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_cb6d6a6fa18e", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 121, "usd": 0.00011544 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_9bc309eb22db", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1377, "completion_tokens": 225, "usd": 0.00013662 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_c0d90d4cb42f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 98, "usd": 0.0000918 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_7d778bb3b477", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 106, "usd": 0.00009492 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1563, "completion_tokens": 226, "usd": 0.0006723 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1433, "completion_tokens": 1437, "usd": 0.005744 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9620dc2201ff", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 119, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00151776, "super": 0.001722, "ultra": 0.012611 }, "counterfactual_ultra_usd": 0.041646, "spent_usd_by_tier": { "nano": 0.00151776, "super": 0.001722, "ultra": 0.013366 }, "prompt_tokens_by_tier": { "nano": 17292, "super": 4147, "ultra": 4384 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 27.103, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_580316692aa6", "verdict_id": "verdict_580316692aa6", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 6 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 6 of 13 replays. Review before merging. The system replayed 13 test pairs against the skill change, which only added a trailing newline to SKILL.md. Seven pairs passed, but six were held as uncertain because the finance.rate tool returned 1.163 for 2026-10-14 instead of the expected 1.08, causing conversion amounts to differ across five expense scenarios. The sixth uncertain pair involved a math calculation unaffected by the code change. No regressions, drifts, or improvements were detected. The cosmetic diff does not explain the rate variance. Investigate whether the finance.rate tool behavior has changed externally or if the golden test data needs updating to match current rates. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_24416e3f8916", "task_id": "fx-no-date", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The replay output uses a different exchange rate (1.1630 vs 1.0800) for the same date (2026-10-14), causing converted amounts to differ (48.85 USD vs 45.36 USD). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1347, "completion_tokens": 209, "usd": 0.00013098 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2378, "completion_tokens": 351, "usd": 0.0010293 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2398, "completion_tokens": 3318, "usd": 0.012352 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_c04a94e117fd", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "drift", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 160, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_823b4ddae43a", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 159, "usd": 0.0001164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_9326a185dca9", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1402, "completion_tokens": 137, "usd": 0.000117 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2585, "completion_tokens": 280, "usd": 0.0010275 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_48d13ccbd36d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 220, "usd": 0.00013494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_24d29ab3083c", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1407, "completion_tokens": 128, "usd": 0.00011514 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_d540794f1c98", "task_id": "fx-coworking-courier", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The replay fails the criterion requiring the rate to be 1.0800 because the finance.rate tool returned 1.163 instead of 1.08 for 2026-10-14. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1426, "completion_tokens": 175, "usd": 0.00012756 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2595, "completion_tokens": 398, "usd": 0.0011367 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2633, "completion_tokens": 1593, "usd": 0.007412 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ecc9e77802a1", "task_id": "fx-software", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The replay violates criteria for USD amount (137.01 vs 127.23), exchange rate (1.1630 vs 1.0800), and total line (137.01 vs 127.23). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1366, "completion_tokens": 170, "usd": 0.00012276 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2410, "completion_tokens": 330, "usd": 0.00102 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2430, "completion_tokens": 1272, "usd": 0.006246 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_ce1c4a74b4df", "task_id": "fx-hamburg-conference", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The replay fails the criterion requiring exchange rate 1 EUR = 1.0800 USD because the finance.rate tool returned 1.163 for 2026-10-14 (t001) instead of 1.08 as in the golden run (t001). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1439, "completion_tokens": 154, "usd": 0.0001233 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2862, "completion_tokens": 367, "usd": 0.0011889 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2877, "completion_tokens": 1652, "usd": 0.007833 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_84352baa167a", "task_id": "fx-berlin-trip", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The finance.rate tool returned a different exchange rate (1.163 vs 1.08) for the same date (2026-10-14) between runs, causing different USD conversions. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 169, "usd": 0.00012594 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2602, "completion_tokens": 356, "usd": 0.001101 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2637, "completion_tokens": 1313, "usd": 0.006576 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_68a265f4d0b3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "drift", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 123, "usd": 0.00009996 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1523, "completion_tokens": 217, "usd": 0.0006522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_eac0515b4a0c", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "drift", "decided_by": "ultra", "reason": "The skill diff only adds a trailing newline to SKILL.md, a cosmetic change that does not alter any step, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 68, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1563, "completion_tokens": 221, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1466, "completion_tokens": 1864, "usd": 0.007058 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_aa0845d7fbd7", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00151092, "super": 0.0078234, "ultra": 0.047477 }, "counterfactual_ultra_usd": 0.096783, "spent_usd_by_tier": { "nano": 0.00151092, "super": 0.0078234, "ultra": 0.048701 }, "prompt_tokens_by_tier": { "nano": 17366, "super": 18518, "ultra": 15152 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 37.407, "simulated_models": false, "repeat": 0, "error": null } ] }, "level0_off": { "config": { "name": "level0_off", "description": "Full cascade without the Level 0 side-effect diff.", "experiments": [ "E5" ], "mode": "edits", "cascade": { "mode": "cascade", "level0": false, "drift": true, "controls": true }, "single_judge_blocks": null, "types": [ "R1", "R2", "R3", "R4", "R5", "R6", "N0", "D1" ], "noise_repeats": 3, "runner": "local", "concurrency": 2, "reuse_replays": "cascade", "max_traces": 40 }, "config_path": "bench/configs/level0_off.yaml", "command": "ratchet bench run --config bench/configs/level0_off.yaml --out bench/results/2026-10-16 --types D1 --limit 2 --yes", "started_at": "2026-10-16T13:06:17.076686Z", "updated_at": "2026-10-16T13:59:07.625370Z", "edits": [ { "bench_id": "n0-github-triage-2", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_e139a48eb217", "verdict_id": "verdict_e139a48eb217", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: github-triage still handles all 13 past tasks.", "narrative": "Passed: github-triage still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated SKILL.md and every pair passed with zero regressions, drift, or uncertain results at a cost of $0.0014. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_811b35141f2c", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 148, "usd": 0.00010458 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_b3d50ce1b799", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1217, "completion_tokens": 133, "usd": 0.00010494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_7741c40ef280", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 131, "usd": 0.00011274 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_abe3d0283a76", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 188, "usd": 0.0001275 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_a0217d5141b7", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 146, "usd": 0.0001149 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_f5d663dbc019", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1345, "completion_tokens": 136, "usd": 0.00011334 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7e7f0890e15f", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1370, "completion_tokens": 174, "usd": 0.00012396 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a1def1b764dc", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 111, "usd": 0.0001074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_0490537a6338", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 175, "usd": 0.0001254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_8b534e552950", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 134, "usd": 0.00011502 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_7e792837459f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 100, "usd": 0.00009228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_95212662e369", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 116, "usd": 0.0000966 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a24da7604ca9", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 109, "usd": 0.00009672 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00143538, "super": 0.0, "ultra": 0.0 }, "counterfactual_ultra_usd": 0.022122, "spent_usd_by_tier": { "nano": 0.00143538, "super": 0.0, "ultra": 0.000405 }, "prompt_tokens_by_tier": { "nano": 16719, "super": 0, "ultra": 225 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 5.121, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-2", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_b1784ae8267e", "verdict_id": "verdict_b1784ae8267e", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The edit changed one line in SKILL.md and every regression pair passed, showing no behavior drift or regressions. The test suite covers the previous task distribution completely with zero failures, uncertain results, or improvements detected. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_db2fa52f33ec", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1106, "completion_tokens": 164, "usd": 0.00010572 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_663c04fc7679", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 195, "usd": 0.00013254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_b8eacf7ca9eb", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1601, "completion_tokens": 116, "usd": 0.0001239 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3975, "completion_tokens": 404, "usd": 0.0015561 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3912, "completion_tokens": 1266, "usd": 0.00771 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_df4a9a6d2d5e", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 185, "usd": 0.00012102 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_2aba1c6b9a48", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 194, "usd": 0.0001323 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_4d8a32096183", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 227, "usd": 0.000141 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_5d79f77bf29a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1315, "completion_tokens": 163, "usd": 0.00011802 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_fa5cfd551a14", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 138, "usd": 0.0001116 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_bcd2f6c4de1b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 127, "usd": 0.00012174 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6c9bfaed6179", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 209, "usd": 0.0001377 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_979bc36816b3", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 147, "usd": 0.00010782 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_0444b760bf07", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 154, "usd": 0.00011322 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_997e188d6e47", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 139, "usd": 0.00010806 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_d40c1503ab73", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 189, "usd": 0.00012096 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_8111d388d469", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 179, "usd": 0.00012 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0727abbc4d2a", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 132, "usd": 0.00010584 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2772, "completion_tokens": 294, "usd": 0.0010962 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_cfff2f4aa409", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 167, "usd": 0.00011502 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_52606712bd67", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 118, "usd": 0.00010422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_542487f91fa5", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 126, "usd": 0.00010518 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2698, "completion_tokens": 309, "usd": 0.0010875 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2657, "completion_tokens": 4091, "usd": 0.01493 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_599436fdbbc6", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_de6195626ea9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1122, "completion_tokens": 81, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1722, "completion_tokens": 221, "usd": 0.0007155 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1609, "completion_tokens": 1313, "usd": 0.005548 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e212cacce84", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 104, "usd": 0.00009618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0025218, "super": 0.0044553, "ultra": 0.028188 }, "counterfactual_ultra_usd": 0.081715, "spent_usd_by_tier": { "nano": 0.0025218, "super": 0.0044553, "ultra": 0.028639 }, "prompt_tokens_by_tier": { "nano": 28614, "super": 11167, "ultra": 8419 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 22.33, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-1", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_5cfd7f483dc4", "verdict_id": "verdict_5cfd7f483dc4", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now fails \"asks for details for #91\" on 4 of 13 past tasks.", "narrative": "Blocked: github-triage now fails \"asks for details for #91\" on 4 of 13 past tasks. The edit removed a single output-formatting line about \"Asked for details\" and did not change any labeling rules, yet four regression pairs show the agent stopped asking for details on specific issues (auth-service #91, mobile-app #64, billing-api #36, widgets #15). One uncertain pair (search-indexer #502) showed a labeling mismatch unrelated to the diff. Eight pairs passed. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_29198b3e615a", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 91, "usd": 0.00009018 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_ae5049772f22", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1175, "completion_tokens": 136, "usd": 0.00010314 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_6a4a7b2c3a20", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails the success criterion 'labels issue #502 as question' (labeled enhancement at step t003), but the skill diff only removes an output-formatting line about 'Asked for details' and does not alter any labeling rules, so the dif…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1315, "completion_tokens": 130, "usd": 0.0001101 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3123, "completion_tokens": 316, "usd": 0.0012213 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3011, "completion_tokens": 3737, "usd": 0.014222 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ff7547adf9e1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1364, "completion_tokens": 177, "usd": 0.00012432 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_95dc2c9a2266", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "asks for details for #91 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1286, "completion_tokens": 174, "usd": 0.00011892 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2979, "completion_tokens": 249, "usd": 0.0011178 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2951, "completion_tokens": 2264, "usd": 0.009743 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_1ac70e25ffb3", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1291, "completion_tokens": 124, "usd": 0.00010722 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3124, "completion_tokens": 320, "usd": 0.0012252 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3015, "completion_tokens": 3102, "usd": 0.012321 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_3be8109cfb2f", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 151, "usd": 0.0001194 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_650e3f7e3ede", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "asks for details on #64 and cites github.comment as artifact changed is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1308, "completion_tokens": 111, "usd": 0.00010512 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3018, "completion_tokens": 262, "usd": 0.0011412 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2909, "completion_tokens": 1234, "usd": 0.006611 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_e5bee22bc04e", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "mentions that issue #36 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1400, "completion_tokens": 96, "usd": 0.00010704 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3411, "completion_tokens": 289, "usd": 0.0012834 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3260, "completion_tokens": 1316, "usd": 0.007208 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_0c922bfa825f", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "mentions #15 was asked for details is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 87, "usd": 0.00010392 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3130, "completion_tokens": 302, "usd": 0.0012108 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3009, "completion_tokens": 1985, "usd": 0.008964 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_00c8cee98bc3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 92, "usd": 0.00009036 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_05517158aa56", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 141, "usd": 0.00010332 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1557, "completion_tokens": 219, "usd": 0.0006642 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1492, "completion_tokens": 1369, "usd": 0.005599 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_bdbe9648c85c", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 122, "usd": 0.00010122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00138426, "super": 0.0078639, "ultra": 0.064668 }, "counterfactual_ultra_usd": 0.11232, "spent_usd_by_tier": { "nano": 0.00138426, "super": 0.0078639, "ultra": 0.065495 }, "prompt_tokens_by_tier": { "nano": 16543, "super": 20342, "ultra": 20117 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 21.669, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_f85a9fa6c1aa", "verdict_id": "verdict_f85a9fa6c1aa", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system replayed 13 test pairs against the updated fx-expense-converter skill; 12 passed and 1 was uncertain. The uncertain case, \"Convert at a weekend date,\" produced a final output date of 2026-10-09 instead of the expected 2026-10-11, and the exchange rate shifted from 1.0768 to 1.0800. The automatic judge could not determine if this deviation is a valid correction or a regression, so it held the change for human review. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_885b30526626", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 212, "usd": 0.0001296 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_fcf301073639", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay violates the criterion 'uses the correct date 2026-10-11' by showing 2026-10-09 in the final output (step m003). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1355, "completion_tokens": 174, "usd": 0.00012306 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2735, "completion_tokens": 281, "usd": 0.0010734 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2683, "completion_tokens": 2822, "usd": 0.011149 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_82a70ccb2af8", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1328, "completion_tokens": 148, "usd": 0.0001152 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_973ca2670ff2", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 102, "usd": 0.0001074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3afca948a379", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 228, "usd": 0.00013686 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_f5be008e5f9d", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1396, "completion_tokens": 142, "usd": 0.00011784 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_b55b161ede07", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1427, "completion_tokens": 148, "usd": 0.00012114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_e70094e8923d", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 214, "usd": 0.00013266 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5bde00727c59", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1404, "completion_tokens": 124, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_96be56214418", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 212, "usd": 0.00013488 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d1a68d58b4ae", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 160, "usd": 0.0001074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_96c18d79778a", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1723, "completion_tokens": 221, "usd": 0.0007158 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1557, "completion_tokens": 1151, "usd": 0.00501 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9cb3c3858cd1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00151914, "super": 0.0017892, "ultra": 0.016159 }, "counterfactual_ultra_usd": 0.045424, "spent_usd_by_tier": { "nano": 0.00156918, "super": 0.0019782, "ultra": 0.016959 }, "prompt_tokens_by_tier": { "nano": 17949, "super": 5043, "ultra": 4587 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 22.796, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-1", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_ecbb5e08c0a5", "verdict_id": "verdict_ecbb5e08c0a5", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now omits issue #75 as documentation on 2 of 13 past tasks.", "narrative": "Blocked: github-triage now omits issue #75 as documentation on 2 of 13 past tasks. The edit changed SKILL.md and triggered regressions on \"Triage acme/infra-terraform\" and \"Triage acme/docs-site\" where required details are no longer captured. Eleven other tasks still pass. The change was reverted automatically. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_f8c354ae33e9", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 134, "usd": 0.00009906 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_478d9ee2ce19", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 133, "usd": 0.00010422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_755deee2dac3", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1429, "completion_tokens": 167, "usd": 0.00012582 } ], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ce4b9acb3157", "task_id": "tri-infra-terraform", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "lists issue #75 as documentation is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 1402, "completion_tokens": 149, "usd": 0.00011988 }, { "tier": "super", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3579, "completion_tokens": 354, "usd": 0.0013923 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 3533, "completion_tokens": 2903, "usd": 0.012242 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 3, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_11ee7c7f4f30", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1336, "completion_tokens": 142, "usd": 0.00011424 } ], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_9432a20fd19d", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1321, "completion_tokens": 167, "usd": 0.00011934 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_715dfb3b7eec", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states that no details were asked for is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 116, "usd": 0.0001107 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3241, "completion_tokens": 273, "usd": 0.001218 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3124, "completion_tokens": 975, "usd": 0.006049 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_5ba0f0c4e92d", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1345, "completion_tokens": 135, "usd": 0.0001131 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3196, "completion_tokens": 294, "usd": 0.0012234 } ], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_39066d273f98", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 183, "usd": 0.00012882 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_c944d9078bcc", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1396, "completion_tokens": 164, "usd": 0.00012312 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_56c2b63686fb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 108, "usd": 0.00009636 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a8e17ca9255f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1616, "completion_tokens": 219, "usd": 0.0006819 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1474, "completion_tokens": 1531, "usd": 0.006067 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_dd5897c02420", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 82, "usd": 0.0000909 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00143232, "super": 0.0045156, "ultra": 0.024358 }, "counterfactual_ultra_usd": 0.06153, "spent_usd_by_tier": { "nano": 0.00143232, "super": 0.0045156, "ultra": 0.024913 }, "prompt_tokens_by_tier": { "nano": 16864, "super": 11632, "ultra": 8446 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 31.397, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-1", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_f6be8d574821", "verdict_id": "verdict_f6be8d574821", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now omits no additional text that is not part of the required output format on 1 of 13 past tasks.", "narrative": "Blocked: github-triage now omits no additional text that is not part of the required output format on 1 of 13 past tasks. The system tested the edit against 13 historical triage tasks and found 11 passing, 1 regression, and 1 uncertain result. The regression on \"Triage acme/status-page\" failed because the output contained extra text beyond the required format at step t002. The uncertain case \"Triage acme/docs-site\" met criteria 1-5 but was held for human review at step t006. The change has been automatically reverted to the previous version. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_165175858316", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 121, "usd": 0.00009882 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_187d5cdcf6dc", "task_id": "tri-status-page", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "contains no additional text that is not part of the required output format is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1228, "completion_tokens": 148, "usd": 0.0001092 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2602, "completion_tokens": 221, "usd": 0.0009795 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2542, "completion_tokens": 901, "usd": 0.005245 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3f0a0fcb0075", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 138, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_9972d203f927", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1361, "completion_tokens": 114, "usd": 0.00010902 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c71e5c2370b8", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 125, "usd": 0.00010752 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_eb9d9e6851a5", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1334, "completion_tokens": 136, "usd": 0.00011268 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_de7759ea9e17", "task_id": "tri-docs-site", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay meets criteria 1-5: it states 4 open issues, lists #210 and #211 as bugs, lists #213 as enhancement, and mentions #214 as already labeled (step m005). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1423, "completion_tokens": 134, "usd": 0.00011754 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3299, "completion_tokens": 274, "usd": 0.0012363 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3250, "completion_tokens": 3381, "usd": 0.013393 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_092b5ebe7778", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1350, "completion_tokens": 155, "usd": 0.0001182 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_968755dee57c", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1413, "completion_tokens": 178, "usd": 0.0001275 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_10dca42d0cf4", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1392, "completion_tokens": 133, "usd": 0.00011544 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 3365, "completion_tokens": 290, "usd": 0.0012705 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_990c76131fc0", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 100, "usd": 0.00009372 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_47c34378e66c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1182, "completion_tokens": 141, "usd": 0.00010476 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_602f5ee8bbac", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1211, "completion_tokens": 100, "usd": 0.00009666 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142578, "super": 0.0034863, "ultra": 0.018638 }, "counterfactual_ultra_usd": 0.052299, "spent_usd_by_tier": { "nano": 0.00142578, "super": 0.0034863, "ultra": 0.019421 }, "prompt_tokens_by_tier": { "nano": 16871, "super": 9266, "ultra": 6167 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 29.262, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_984c92fa365f", "verdict_id": "verdict_984c92fa365f", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 14 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 14 replays. Review before merging. The skill diff only adds the word \"expense\" to the description and rewords another description, changing no tool, argument, step, or output rule. Twelve replays passed. Two replays were marked uncertain: \"Convert at a weekend date\" showed a Tavily value shift from 1.0768 to 1.0800, and \"Rewrite a sentence formally\" had no logic change. No regressions, drift, or improvements were found. Cost was negligible. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_9b9fe0986841", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1324, "completion_tokens": 227, "usd": 0.00013392 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_f97cae02fbc4", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds the word 'expense' to the description field, which does not change any tool, argument, step, or output rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1352, "completion_tokens": 147, "usd": 0.0001164 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2555, "completion_tokens": 283, "usd": 0.0010212 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2496, "completion_tokens": 1930, "usd": 0.008286 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6eb01a2d41d3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1342, "completion_tokens": 159, "usd": 0.00011868 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_315104ec0683", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 89, "usd": 0.00010428 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_762ca612dbe8", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 262, "usd": 0.0001443 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_4ef2ca8f8b72", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1396, "completion_tokens": 202, "usd": 0.00013224 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_53e1872e54f8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 135, "usd": 0.0001188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_fddd61f372fe", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 165, "usd": 0.00011874 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_bda2112c7543", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 119, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c3051db65b47", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 173, "usd": 0.00012624 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_8981ccff882f", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1602, "completion_tokens": 128, "usd": 0.00012684 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3794, "completion_tokens": 395, "usd": 0.0014937 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3750, "completion_tokens": 2079, "usd": 0.009987 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_ec291cce2b97", "task_id": "canary-formal-rewrite", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only rewords the description and does not affect the prompt or generation logic. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 116, "usd": 0.00009708 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1514, "completion_tokens": 266, "usd": 0.0006936 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1497, "completion_tokens": 1440, "usd": 0.005817 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_cbb67ab2491c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1533, "completion_tokens": 214, "usd": 0.0006525 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1387, "completion_tokens": 1109, "usd": 0.004714 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_91fcb24262ed", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 99, "usd": 0.0000957 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00163638, "super": 0.003861, "ultra": 0.028804 }, "counterfactual_ultra_usd": 0.066854, "spent_usd_by_tier": { "nano": 0.00168642, "super": 0.0040509, "ultra": 0.029568 }, "prompt_tokens_by_tier": { "nano": 19603, "super": 9984, "ultra": 9501 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 16.176, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_4af389711029", "verdict_id": "verdict_4af389711029", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs against the updated fx-expense-converter skill and found 12 passing with no regressions, drift, or improvements. One case, \"Compute 17% of 240,\" produced an uncertain result because the skill changes did not explain the output difference, suggesting possible nondeterminism. That specific replay was held for human review. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_382f2e4d44ae", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1300, "completion_tokens": 110, "usd": 0.0001044 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_75e2cc6a76a8", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 160, "usd": 0.00011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_c536f0cd0389", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 161, "usd": 0.0001176 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_a8c493d9890b", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 102, "usd": 0.0001074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_b7a6a78b225c", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1345, "completion_tokens": 208, "usd": 0.00013062 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a00a986b53f7", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1383, "completion_tokens": 135, "usd": 0.00011538 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_c7bcfb7bf68e", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1428, "completion_tokens": 179, "usd": 0.00012864 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_3b832d8b5c25", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1367, "completion_tokens": 160, "usd": 0.00012042 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_f76c8a253ef0", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 127, "usd": 0.0001176 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_c432f688c91f", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 189, "usd": 0.00012936 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1f1fa89dddfb", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 105, "usd": 0.0000942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_9470f1a93d4f", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 74, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1592, "completion_tokens": 245, "usd": 0.0006981 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1521, "completion_tokens": 1746, "usd": 0.006759 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f86b4f70f4f6", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 106, "usd": 0.00009522 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00146616, "super": 0.0006981, "ultra": 0.006759 }, "counterfactual_ultra_usd": 0.031706, "spent_usd_by_tier": { "nano": 0.00146616, "super": 0.0006981, "ultra": 0.007391 }, "prompt_tokens_by_tier": { "nano": 17172, "super": 1592, "ultra": 1820 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 10.329, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-expense-report-1", "skill": "expense-report", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_4758efe03506", "verdict_id": "verdict_4758efe03506", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 22 replays. Review before merging. The system re-ran 22 test pairs against the expense-report skill change; 21 passed and 1 was marked uncertain. The uncertain case, \"Clean support/tickets.csv,\" showed a difference the skill diff does not explain, suggesting nondeterminism. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_ddb84b0fe47b", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1178, "completion_tokens": 122, "usd": 0.00009996 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_7d67f2353f20", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1407, "completion_tokens": 159, "usd": 0.00012258 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_acc26084d722", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1622, "completion_tokens": 132, "usd": 0.000129 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3923, "completion_tokens": 134, "usd": 0.0012975 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3914, "completion_tokens": 1311, "usd": 0.007847 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_af5ade88f377", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 237, "usd": 0.00013494 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_a69ed051d7f4", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1441, "completion_tokens": 142, "usd": 0.00012054 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_3e7d7797755d", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1430, "completion_tokens": 195, "usd": 0.0001326 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_079c36312676", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1303, "completion_tokens": 170, "usd": 0.00011898 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_f07720304e5f", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1296, "completion_tokens": 216, "usd": 0.0001296 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_8626f6fe5abb", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1521, "completion_tokens": 155, "usd": 0.00012846 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_521e59e6091f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1435, "completion_tokens": 140, "usd": 0.0001197 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_e31167be7710", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 198, "usd": 0.00012006 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_229e9b0e57cc", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 107, "usd": 0.00010194 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_1aec4a330ef6", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 113, "usd": 0.00010182 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_5b8721009163", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 162, "usd": 0.00011322 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3aee4f6f9c1d", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 105, "usd": 0.00010224 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2742, "completion_tokens": 280, "usd": 0.0010746 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2681, "completion_tokens": 1819, "usd": 0.008138 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_838608cb3f0e", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 158, "usd": 0.00011496 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2693, "completion_tokens": 285, "usd": 0.0010644 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_010f6ae26a1d", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 146, "usd": 0.0001107 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2252, "completion_tokens": 265, "usd": 0.0009141 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_a8ba615a5d23", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1313, "completion_tokens": 172, "usd": 0.00012006 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_22e0e0ff85d9", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 159, "usd": 0.0001131 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2774, "completion_tokens": 324, "usd": 0.0011238 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2669, "completion_tokens": 3979, "usd": 0.014606 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_15046a030360", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3ac334e2f481", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 186, "usd": 0.00011412 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1621, "completion_tokens": 286, "usd": 0.0007437 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 1614, "completion_tokens": 970, "usd": 0.004524 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb845c98bd90", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1175, "completion_tokens": 105, "usd": 0.0000957 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00253728, "super": 0.0062181, "ultra": 0.035115 }, "counterfactual_ultra_usd": 0.094751, "spent_usd_by_tier": { "nano": 0.00253728, "super": 0.0062181, "ultra": 0.03568 }, "prompt_tokens_by_tier": { "nano": 28772, "super": 16005, "ultra": 11164 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 21.731, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-expense-report-1", "skill": "expense-report", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_e7c89dbb45dc", "verdict_id": "verdict_e7c89dbb45dc", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: expense-report now omits the VAT amount '7.58 EUR' on 3 of 22 past tasks.", "narrative": "Blocked: expense-report now omits the VAT amount '7.58 EUR' on 3 of 22 past tasks. The evaluation ran 22 regression pairs and found 3 failures where the agent stopped extracting the merchant, date, total, and VAT values for office supplies, conference, and lunch and train receipts. The remaining 19 pairs passed. The edit changed SKILL.md with one addition and two deletions. The regressions indicate the update broke existing VAT extraction logic. Roll back the change and fix the skill so it preserves the required fields before re-submitting. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_73fd927ba82e", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1154, "completion_tokens": 107, "usd": 0.00009492 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_71599b6cf2f5", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 197, "usd": 0.00013122 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_68029dbf8d42", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1597, "completion_tokens": 155, "usd": 0.00013302 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3826, "completion_tokens": 431, "usd": 0.0015357 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.75, "prompt_tokens": 3810, "completion_tokens": 2530, "usd": 0.0114 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_0061479721a1", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1293, "completion_tokens": 125, "usd": 0.00010758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_cde84ba8b5d5", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1427, "completion_tokens": 152, "usd": 0.0001221 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_d2bfe3e90095", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1415, "completion_tokens": 139, "usd": 0.00011826 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_ea8d1c2b4f4b", "task_id": "exp-office-supplies", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the VAT amount '7.58 EUR' is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 1284, "completion_tokens": 83, "usd": 0.00009696 }, { "tier": "super", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 2509, "completion_tokens": 275, "usd": 0.0010002 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2381, "completion_tokens": 791, "usd": 0.004754 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_bcff8a71d40d", "task_id": "exp-conference", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the VAT as 79.83 EUR is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1312, "completion_tokens": 172, "usd": 0.00012 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2525, "completion_tokens": 275, "usd": 0.001005 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2477, "completion_tokens": 977, "usd": 0.005408 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_49a1a41ef5ee", "task_id": "exp-lunch-train", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1507, "completion_tokens": 130, "usd": 0.00012162 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3382, "completion_tokens": 384, "usd": 0.0013602 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3325, "completion_tokens": 1371, "usd": 0.007438 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_87ffd7a9b61f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1444, "completion_tokens": 142, "usd": 0.00012072 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_bfab509e3676", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1257, "completion_tokens": 165, "usd": 0.00011502 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_ae36c7d648f7", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 107, "usd": 0.00009978 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_c94efe16dd1d", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 198, "usd": 0.00012222 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ffd4fd935f19", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1263, "completion_tokens": 134, "usd": 0.00010794 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_699475671d46", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 175, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_bbc5ac91d665", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 162, "usd": 0.0001152 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2718, "completion_tokens": 306, "usd": 0.0010908 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_657e87d6ef5e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 191, "usd": 0.0001215 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_8d54d0935d9d", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 194, "usd": 0.0001239 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_cbe81857d988", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 184, "usd": 0.00011838 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2578, "completion_tokens": 327, "usd": 0.0010677 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2581, "completion_tokens": 2092, "usd": 0.008857 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_db46f3b0746f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 141, "usd": 0.00010212 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_10a2feb40124", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 80, "usd": 0.00008868 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1627, "completion_tokens": 214, "usd": 0.0006807 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1515, "completion_tokens": 1022, "usd": 0.004581 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_da322788c4ba", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 103, "usd": 0.00009666 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00249612, "super": 0.0077403, "ultra": 0.042438 }, "counterfactual_ultra_usd": 0.106605, "spent_usd_by_tier": { "nano": 0.00249612, "super": 0.0077403, "ultra": 0.043176 }, "prompt_tokens_by_tier": { "nano": 28658, "super": 19165, "ultra": 16449 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.576, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r4-github-triage-2", "skill": "github-triage", "type": "R4", "expected_decision": "BLOCK", "edit_id": "edit_ea6a21b76649", "verdict_id": "verdict_ea6a21b76649", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The edit changed SKILL.md for github-triage. Thirteen replays ran: twelve passed, one was uncertain. The uncertain replay for \"Triage acme/mobile-app\" violated two success criteria by failing to label issue #58 as a bug and incorrectly listing it under 'Already labeled'. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0a31fbd0f6ab", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1139, "completion_tokens": 117, "usd": 0.00009642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e3ade3806218", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1185, "completion_tokens": 149, "usd": 0.00010686 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e7ec930d750a", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1359, "completion_tokens": 176, "usd": 0.00012378 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_c0c9ec6a7dc8", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1353, "completion_tokens": 97, "usd": 0.00010446 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3257, "completion_tokens": 267, "usd": 0.0012174 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_9484d74e29ab", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1330, "completion_tokens": 173, "usd": 0.00012132 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_dcfc05cb8a6e", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1365, "completion_tokens": 181, "usd": 0.00012534 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3099, "completion_tokens": 319, "usd": 0.0012168 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_b752b4bba738", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 175, "usd": 0.00012738 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_811329a74d4f", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay violates two success criteria: it fails to label #58 as bug and incorrectly lists #58 under 'Already labeled'. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1319, "completion_tokens": 125, "usd": 0.00010914 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2987, "completion_tokens": 308, "usd": 0.0011733 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2862, "completion_tokens": 1401, "usd": 0.007065 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9dd1c1c9ae58", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 102, "usd": 0.00010734 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3625, "completion_tokens": 295, "usd": 0.001353 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d9834ffa3e3a", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1463, "completion_tokens": 119, "usd": 0.00011634 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3f0ff18aec4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 125, "usd": 0.000099 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_5b3dba8d5830", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1537, "completion_tokens": 243, "usd": 0.0006798 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1488, "completion_tokens": 1081, "usd": 0.004731 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f28450be2e37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 132, "usd": 0.00010362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142848, "super": 0.0056403, "ultra": 0.011796 }, "counterfactual_ultra_usd": 0.052662, "spent_usd_by_tier": { "nano": 0.00142848, "super": 0.0056403, "ultra": 0.012396 }, "prompt_tokens_by_tier": { "nano": 16836, "super": 14505, "ultra": 4653 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.551, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-2", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_38da6176e946", "verdict_id": "verdict_38da6176e946", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The skill edit adds a branch parameter to the github.list_issues call. In the uncertain replay for acme/billing-api, the tool call omitted this new parameter yet returned identical issue data to the golden run. All other 12 replays passed with no regressions or drift. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5bbf428edf52", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 115, "usd": 0.00009564 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1606, "completion_tokens": 177, "usd": 0.0006411 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f2d56880f7aa", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 169, "usd": 0.00011286 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_c651c351049c", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 180, "usd": 0.00012624 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_b6ff0cb2781b", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1418, "completion_tokens": 206, "usd": 0.00013452 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3670, "completion_tokens": 320, "usd": 0.001389 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3658, "completion_tokens": 3353, "usd": 0.013717 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac63d98c8094", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1314, "completion_tokens": 125, "usd": 0.00010884 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_537a1694bbb0", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1357, "completion_tokens": 206, "usd": 0.00013086 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_9e2fa3314653", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1490, "completion_tokens": 178, "usd": 0.00013212 } ], "control_outcome": null, "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d200a62ffe51", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 139, "usd": 0.00011412 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_465ab5c43f9c", "task_id": "tri-billing-api", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only adds a 'branch' parameter to the github.list_issues call, but the replay run's tool call (t001) omits this parameter and returns the same issue data as the golden run. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1351, "completion_tokens": 98, "usd": 0.00010458 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3319, "completion_tokens": 340, "usd": 0.0013017 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3290, "completion_tokens": 3126, "usd": 0.012668 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_19dcfee12b9c", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 127, "usd": 0.00011448 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f59a42a6f25f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 97, "usd": 0.00009228 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f5acfa748023", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1626, "completion_tokens": 222, "usd": 0.0006876 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1502, "completion_tokens": 3943, "usd": 0.013331 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_de28baa02c6f", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1211, "completion_tokens": 101, "usd": 0.0000969 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014502, "super": 0.0040194, "ultra": 0.039716 }, "counterfactual_ultra_usd": 0.075471, "spent_usd_by_tier": { "nano": 0.0014502, "super": 0.0040194, "ultra": 0.040322 }, "prompt_tokens_by_tier": { "nano": 16918, "super": 10221, "ultra": 8771 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.684, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_7bd4935e285d", "verdict_id": "verdict_7bd4935e285d", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system replayed 13 test pairs against the skill change; 12 passed and 1 was uncertain. The uncertain case, \"Compute 17% of 240,\" showed a difference the skill diff does not explain, likely due to nondeterminism. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_bf3fa0074e07", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 135, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_e649669b0a50", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 119, "usd": 0.00010608 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_6b67a590cae3", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1340, "completion_tokens": 167, "usd": 0.00012048 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_1f156a1c8f92", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1370, "completion_tokens": 102, "usd": 0.00010668 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_3e98cef87a9d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1357, "completion_tokens": 182, "usd": 0.0001251 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9ffedc5fd087", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1372, "completion_tokens": 152, "usd": 0.0001188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_941456f71c6b", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1439, "completion_tokens": 138, "usd": 0.00011946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_4edc319d8099", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 196, "usd": 0.00012834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_5ece1e8c9118", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 119, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_400354316051", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1388, "completion_tokens": 163, "usd": 0.0001224 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_6cfa1d232eb4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_13feb5997c98", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 58, "usd": 0.00008412 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1543, "completion_tokens": 218, "usd": 0.0006591 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 1814, "usd": 0.006802 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3e2f6c6c99b4", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 99, "usd": 0.0000957 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00144648, "super": 0.0006591, "ultra": 0.006802 }, "counterfactual_ultra_usd": 0.031379, "spent_usd_by_tier": { "nano": 0.00144648, "super": 0.0006591, "ultra": 0.007365 }, "prompt_tokens_by_tier": { "nano": 17196, "super": 1543, "ultra": 1659 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 9.995, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-expense-report-1", "skill": "expense-report", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_c11a868df7ca", "verdict_id": "verdict_c11a868df7ca", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The evaluation ran 22 regression pairs against the updated skill and every pair passed with zero regressions, drift, or uncertain results. The change modified SKILL.md with a minimal one-line addition and one-line removal. Execution cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_60c12e465a52", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1129, "completion_tokens": 130, "usd": 0.00009894 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 1601, "completion_tokens": 149, "usd": 0.0006144 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_87b3a1580656", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1386, "completion_tokens": 117, "usd": 0.00011124 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_ac01bb0f765a", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1566, "completion_tokens": 100, "usd": 0.00011796 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3794, "completion_tokens": 419, "usd": 0.0015153 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3757, "completion_tokens": 1308, "usd": 0.007681 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_ba09cf46ac3d", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1277, "completion_tokens": 103, "usd": 0.00010134 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_ca6090d7b4f3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1453, "completion_tokens": 104, "usd": 0.00011214 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_f88f40b0fdd4", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 184, "usd": 0.00013068 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_4c04fd872a46", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1303, "completion_tokens": 180, "usd": 0.00012138 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a5beee38284e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1308, "completion_tokens": 142, "usd": 0.00011256 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37245d26f903", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1509, "completion_tokens": 161, "usd": 0.00012918 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_4e2b952bb502", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1447, "completion_tokens": 135, "usd": 0.00011922 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_803282d49f8b", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 145, "usd": 0.00010878 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_c5f95c50e3d5", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1223, "completion_tokens": 162, "usd": 0.00011226 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_8a5a0ea0ed7f", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 163, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_812811600cab", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 106, "usd": 0.00009942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_eb78878a2ed4", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1269, "completion_tokens": 181, "usd": 0.00011958 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2855, "completion_tokens": 278, "usd": 0.0011067 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_0932e89999f7", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 176, "usd": 0.00011784 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2684, "completion_tokens": 307, "usd": 0.0010815 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2654, "completion_tokens": 2197, "usd": 0.009245 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_154346b1b35e", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 162, "usd": 0.00011526 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_388513c72ff0", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1361, "completion_tokens": 174, "usd": 0.00012342 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_1d8ff02cb8d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1261, "completion_tokens": 118, "usd": 0.00010398 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2603, "completion_tokens": 292, "usd": 0.0010437 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2522, "completion_tokens": 2893, "usd": 0.011201 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_43b809e3abdd", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 112, "usd": 0.0000966 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_62160dffa87e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 141, "usd": 0.00010404 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f2e5587a73ca", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 109, "usd": 0.00009744 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00246798, "super": 0.0053616, "ultra": 0.028127 }, "counterfactual_ultra_usd": 0.084027, "spent_usd_by_tier": { "nano": 0.00246798, "super": 0.0053616, "ultra": 0.028584 }, "prompt_tokens_by_tier": { "nano": 28713, "super": 13537, "ultra": 9174 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.456, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-2", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_0ca8e3d75ad9", "verdict_id": "verdict_0ca8e3d75ad9", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system re-ran 13 test pairs against the csv-cleaner skill change and found 12 passing with one uncertain result on a percentage calculation where the skill diff did not explain the variance. The uncertain case was held for human review due to likely nondeterminism. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_959e7e0816aa", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 230, "usd": 0.00012942 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2239, "completion_tokens": 294, "usd": 0.0009363 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_b81786a373a5", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 158, "usd": 0.00011298 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_1dca9722e993", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1209, "completion_tokens": 157, "usd": 0.00011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_96dcc8778ae9", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 194, "usd": 0.00012288 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_130fa3316eae", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 87, "usd": 0.0000972 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2722, "completion_tokens": 295, "usd": 0.0010821 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2554, "completion_tokens": 2334, "usd": 0.009556 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_b7dba255cdef", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1265, "completion_tokens": 128, "usd": 0.00010662 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_fe6d06e8fe47", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 169, "usd": 0.00011466 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_5d23e89670d4", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 117, "usd": 0.0001023 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2606, "completion_tokens": 335, "usd": 0.0010833 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 5732, "completion_tokens": 6180, "usd": 0.024272000000000002 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_02fdaa1af68e", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 182, "usd": 0.00012174 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3336, "completion_tokens": 319, "usd": 0.0012879 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7c626553532c", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 159, "usd": 0.00011286 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8b65fb6e74ff", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_f33745e12a8e", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 178, "usd": 0.0001122 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1569, "completion_tokens": 219, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1527, "completion_tokens": 1644, "usd": 0.006459 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_690661728b24", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 82, "usd": 0.00008946 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142554, "super": 0.0050574, "ultra": 0.040287 }, "counterfactual_ultra_usd": 0.078963, "spent_usd_by_tier": { "nano": 0.00142554, "super": 0.0050574, "ultra": 0.040867 }, "prompt_tokens_by_tier": { "nano": 15995, "super": 12472, "ultra": 10108 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 32.038, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-2", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_e864035b7dff", "verdict_id": "verdict_e864035b7dff", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now fails \"does not ask for details for #90\" on 5 of 13 past tasks.", "narrative": "Blocked: github-triage now fails \"does not ask for details for #90\" on 5 of 13 past tasks. The edit changed one line in SKILL.md and caused regressions across five repositories including auth-service, docs-site, mobile-app, billing-api, and widgets. Failures include missing detail requests, incorrect artifact citations, missing issue labels, and erroneous comments on enhancement issues. All 13 evaluation pairs were re-run with 8 passing and 5 regressing. The system correctly reverted the change. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_aeb36d32b26b", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1139, "completion_tokens": 147, "usd": 0.00010362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_fb3621bc1546", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 138, "usd": 0.00010542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_a026774bb5dc", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1348, "completion_tokens": 138, "usd": 0.000114 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_082c7d58917e", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 204, "usd": 0.00013332 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3788, "completion_tokens": 316, "usd": 0.0014208 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3741, "completion_tokens": 1756, "usd": 0.009009 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_e82538fe8409", "task_id": "tri-auth-service", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "does not ask for details for #90 is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 1341, "completion_tokens": 112, "usd": 0.00010734 }, { "tier": "super", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3347, "completion_tokens": 337, "usd": 0.0013074 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3236, "completion_tokens": 1275, "usd": 0.007061 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_4281d13e3b01", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1343, "completion_tokens": 113, "usd": 0.0001077 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3396, "completion_tokens": 386, "usd": 0.0013662 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3341, "completion_tokens": 1578, "usd": 0.008075 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_7ee8c01d51f9", "task_id": "tri-docs-site", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states that no details were asked for is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1388, "completion_tokens": 129, "usd": 0.00011424 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3188, "completion_tokens": 297, "usd": 0.0012237 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.98, "prompt_tokens": 3142, "completion_tokens": 703, "usd": 0.005251 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4f8712b7b45d", "task_id": "tri-mobile-app", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "asks for details on #64 and cites github.comment as artifact changed is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 1384, "completion_tokens": 116, "usd": 0.00011088 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3400, "completion_tokens": 343, "usd": 0.0013287 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.85, "prompt_tokens": 3330, "completion_tokens": 5015, "usd": 0.018375 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_650cb16fa7da", "task_id": "tri-billing-api", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "lists the issue numbers and their labels is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 1441, "completion_tokens": 161, "usd": 0.0001251 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3969, "completion_tokens": 385, "usd": 0.0015372 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3942, "completion_tokens": 2053, "usd": 0.010101 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_6f78304e2ae4", "task_id": "tri-widgets", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff changed step 6 to also comment on enhancement issues, causing the replay to attempt commenting on #7 (labeled enhancement) and incorrectly report it in the summary is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1425, "completion_tokens": 161, "usd": 0.00012414 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3702, "completion_tokens": 312, "usd": 0.0013914 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3595, "completion_tokens": 943, "usd": 0.006424 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_96b50023c3c8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 112, "usd": 0.00009588 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_575a1d924d31", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1170, "completion_tokens": 173, "usd": 0.00011172 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8419e2ca5496", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 100, "usd": 0.00009594 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014493, "super": 0.0095754, "ultra": 0.064296 }, "counterfactual_ultra_usd": 0.118565, "spent_usd_by_tier": { "nano": 0.0014493, "super": 0.0095754, "ultra": 0.065085 }, "prompt_tokens_by_tier": { "nano": 16939, "super": 24790, "ultra": 24774 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 27.996, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-github-triage-2", "skill": "github-triage", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_54289ce1adbf", "verdict_id": "verdict_54289ce1adbf", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now drops the phrase \"= **40.8**\" on 1 of 13 past tasks.", "narrative": "Blocked: github-triage now drops the phrase \"= **40.8**\" on 1 of 13 past tasks. The edit added two issue-closure rules to SKILL.md and was tested against 13 prior tasks. Eleven passed, one regressed on a math task where the exact output \"= **40.8**\" disappeared, one was held for human review because the new rules were irrelevant to the task, and the rest passed. The regression indicates the diff introduced a side effect on unrelated behavior. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_5da690453eaf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 145, "usd": 0.00010386 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_66be25a1b831", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1206, "completion_tokens": 150, "usd": 0.00010836 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_9803f74fad28", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff adds only two rules about closing issues, which are irrelevant to the task (no issue closure requested). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1356, "completion_tokens": 147, "usd": 0.00011664 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3187, "completion_tokens": 324, "usd": 0.0012477 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3164, "completion_tokens": 3607, "usd": 0.013985 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 1 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_287402e48fdd", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1337, "completion_tokens": 176, "usd": 0.00012246 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ad012c68b2ce", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1290, "completion_tokens": 113, "usd": 0.00010452 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_c31ee7fd6635", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1364, "completion_tokens": 114, "usd": 0.0001092 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3076, "completion_tokens": 331, "usd": 0.0012207 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_a3e558951509", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1430, "completion_tokens": 191, "usd": 0.00013164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7af9bb18724a", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1345, "completion_tokens": 116, "usd": 0.00010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_9ea5f7e643be", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1390, "completion_tokens": 177, "usd": 0.00012588 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_acc745f38e82", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 214, "usd": 0.00013608 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_e3ac050839ba", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 113, "usd": 0.0000954 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1e223b9eccf6", "task_id": "canary-percent", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "output includes the phrase \"= **40.8**\" is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 81, "usd": 0.00008964 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1573, "completion_tokens": 226, "usd": 0.0006753 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1456, "completion_tokens": 1971, "usd": 0.007369 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_a03ddffaa752", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1211, "completion_tokens": 82, "usd": 0.00009234 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00144456, "super": 0.0031437, "ultra": 0.021354 }, "counterfactual_ultra_usd": 0.05409, "spent_usd_by_tier": { "nano": 0.00144456, "super": 0.0031437, "ultra": 0.022035 }, "prompt_tokens_by_tier": { "nano": 16800, "super": 7836, "ultra": 4962 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.855, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-github-triage-1", "skill": "github-triage", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_ed35d900fd97", "verdict_id": "verdict_ed35d900fd97", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The skill edit only updates SKILL.md to document the github.label parameter as a string instead of an array, but both the baseline and candidate runs used the old array format in actual tool calls. Twelve replays passed and one was marked uncertain because the judge could not determine if the behavior change matters. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_bd63d6f067f1", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1115, "completion_tokens": 121, "usd": 0.00009594 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_f6019126d256", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1181, "completion_tokens": 108, "usd": 0.00009678 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_dd87261b8ba6", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 164, "usd": 0.0001224 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_2d58c0602c67", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1368, "completion_tokens": 136, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_c9de1093aa99", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1307, "completion_tokens": 156, "usd": 0.00011586 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5101ae1012ad", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1345, "completion_tokens": 209, "usd": 0.00013086 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_47ce9e153501", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1393, "completion_tokens": 200, "usd": 0.00013158 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_4db894881c54", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1322, "completion_tokens": 99, "usd": 0.00010308 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_c8c4828aae44", "task_id": "tri-billing-api", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only changes the documented github.label parameter from 'labels' array to 'label' string in SKILL.md; both runs actually used the old 'labels' array format in tool calls. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1409, "completion_tokens": 172, "usd": 0.00012582 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3536, "completion_tokens": 339, "usd": 0.0013659 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3514, "completion_tokens": 2966, "usd": 0.012412 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_b3c44e403f10", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 153, "usd": 0.00012072 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f3095a10d7d9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 98, "usd": 0.00009252 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_4519fd683ef9", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1629, "completion_tokens": 219, "usd": 0.0006858 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1523, "completion_tokens": 2002, "usd": 0.007529 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_72d1ef164d37", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 119, "usd": 0.0001005 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00143682, "super": 0.0020517, "ultra": 0.019941 }, "counterfactual_ultra_usd": 0.04892, "spent_usd_by_tier": { "nano": 0.00143682, "super": 0.0020517, "ultra": 0.020547 }, "prompt_tokens_by_tier": { "nano": 16719, "super": 5165, "ultra": 5352 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.167, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_3e5c093ac8e4", "verdict_id": "verdict_3e5c093ac8e4", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system replayed 13 test cases against the updated fx-expense-converter skill; 12 passed and 1 was uncertain. The uncertain case, \"Convert at a weekend date,\" returned a rate dated 2026-10-09 instead of the requested 2026-10-11, and Tavily detected the rate value shifting from 1.0768 to 1.0800. No regressions, drift, or improvements were found. The change cost a fraction of a cent to evaluate. Check whether the weekend date fallback to the prior Friday is intended behavior. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e5b902ded754", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 135, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_8c4cb14a441e", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output shows the rate date as 2026-10-09 instead of the requested 2026-10-11, violating the 'uses the correct date 2026-10-11' criterion. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 117, "usd": 0.00010848 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2627, "completion_tokens": 287, "usd": 0.0010464 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2635, "completion_tokens": 2752, "usd": 0.010891 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_3f28b051e158", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1328, "completion_tokens": 181, "usd": 0.00012312 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_bdb089b8bdbb", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1346, "completion_tokens": 94, "usd": 0.00010332 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_2393bfd5f040", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1393, "completion_tokens": 232, "usd": 0.00013926 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_a029bb9d7e18", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1371, "completion_tokens": 183, "usd": 0.00012618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_148da7df7bf5", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1452, "completion_tokens": 204, "usd": 0.00013608 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_c24c16f3b5fa", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1331, "completion_tokens": 186, "usd": 0.0001245 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_b4e1ea78a932", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 122, "usd": 0.00011496 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_cd2b69e99407", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 132, "usd": 0.0001164 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_83397ea2c263", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 108, "usd": 0.0000942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_37f7a8b8bf4c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 142, "usd": 0.00010428 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1614, "completion_tokens": 229, "usd": 0.0006903 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1586, "completion_tokens": 1326, "usd": 0.005564 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_8108d03dbe59", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1176, "completion_tokens": 115, "usd": 0.00009816 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00150006, "super": 0.0017367, "ultra": 0.016455 }, "counterfactual_ultra_usd": 0.045294, "spent_usd_by_tier": { "nano": 0.00155034, "super": 0.0019266, "ultra": 0.017297 }, "prompt_tokens_by_tier": { "nano": 17899, "super": 4829, "ultra": 4577 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.097, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-1", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_f2278045b8ba", "verdict_id": "verdict_f2278045b8ba", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: github-triage still handles all 13 past tasks.", "narrative": "Passed: github-triage still handles all 13 past tasks. The evaluation ran 13 regression pairs against the updated SKILL.md and every pair passed with zero regressions, drift, or uncertain results. The change modified one line in the skill file and the agent behavior remained consistent across all known scenarios. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_36523cbf4c92", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1127, "completion_tokens": 150, "usd": 0.00010362 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e70cb39d820f", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 160, "usd": 0.0001107 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_93bcdbcd3914", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 108, "usd": 0.00010722 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_ba25466de412", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1385, "completion_tokens": 177, "usd": 0.00012558 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_eb462443ad9b", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1295, "completion_tokens": 146, "usd": 0.00011274 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5d5629ab1937", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1343, "completion_tokens": 110, "usd": 0.00010698 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3054, "completion_tokens": 312, "usd": 0.001197 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_37a737ab09a1", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 177, "usd": 0.00012684 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_a8f42769e490", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1329, "completion_tokens": 139, "usd": 0.0001131 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dbbba87c46a3", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1401, "completion_tokens": 176, "usd": 0.0001263 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_84d120ce3156", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 119, "usd": 0.00011256 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_f1c02a8e442b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 87, "usd": 0.0000906 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_227ef3c2f411", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 79, "usd": 0.00008772 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1560, "completion_tokens": 213, "usd": 0.0006597 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1444, "completion_tokens": 1241, "usd": 0.005167 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_ed4fe7d3e21d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1187, "completion_tokens": 112, "usd": 0.0000981 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142206, "super": 0.0018567, "ultra": 0.005167 }, "counterfactual_ultra_usd": 0.033317, "spent_usd_by_tier": { "nano": 0.00142206, "super": 0.0018567, "ultra": 0.005644 }, "prompt_tokens_by_tier": { "nano": 16741, "super": 4614, "ultra": 1690 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 9.197, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_90f0de5a0c20", "verdict_id": "verdict_90f0de5a0c20", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The regression suite ran 13 pairs against the updated SKILL.md and every case returned the expected result with zero regressions, drift, or uncertain outcomes. The change modified only the skill documentation by one line, leaving the core conversion logic untouched. Test coverage spans the standard currency and formatting scenarios the skill is designed for. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_8c2c5e26d4e1", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 129, "usd": 0.00011112 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_2a31e3e43325", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1341, "completion_tokens": 208, "usd": 0.00013038 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_40498db1a5f2", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 197, "usd": 0.00012624 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_0f0b98c0dda1", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1536, "completion_tokens": 164, "usd": 0.00013152 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_a341fb0948fa", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 221, "usd": 0.00013518 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_b5a453299350", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1383, "completion_tokens": 207, "usd": 0.00013266 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ea5e1042bbf8", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1427, "completion_tokens": 135, "usd": 0.00011802 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ab59cb62584a", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 223, "usd": 0.00013266 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_d101d55224e4", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1416, "completion_tokens": 124, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_f96f5f3cef70", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1388, "completion_tokens": 163, "usd": 0.0001224 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_eb8645af7d9f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 106, "usd": 0.00009588 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_710563b99aef", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 80, "usd": 0.00008796 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1685, "completion_tokens": 224, "usd": 0.0007071 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 1592, "completion_tokens": 1725, "usd": 0.006767 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_fb5c11b812f5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 131, "usd": 0.00010272 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1664, "completion_tokens": 239, "usd": 0.0007143 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00154146, "super": 0.0014214, "ultra": 0.006767 }, "counterfactual_ultra_usd": 0.035108, "spent_usd_by_tier": { "nano": 0.00154146, "super": 0.0014214, "ultra": 0.007307 }, "prompt_tokens_by_tier": { "nano": 17339, "super": 3349, "ultra": 1844 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 12.538, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-2", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_82614c5f6461", "verdict_id": "verdict_82614c5f6461", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 1 of 22 past tasks.", "narrative": "Blocked: expense-report now omits the merchant, date, total and VAT for each of the 2 receipts on 1 of 22 past tasks. The system ran 22 regression pairs against the previous version and found 21 passing and 1 regression where the agent failed to extract required fields for a bakery receipt scenario. The change was automatically reverted. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_efb6b4383a27", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1149, "completion_tokens": 111, "usd": 0.00009558 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_26a650f173a0", "task_id": "exp-missing-receipt", "outcome": "regression", "expected_outcome": "pass", "decided_by": "ultra", "reason": "states the merchant, date, total and VAT for each of the 2 receipts is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1429, "completion_tokens": 186, "usd": 0.00013038 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3424, "completion_tokens": 316, "usd": 0.0013116 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3409, "completion_tokens": 2189, "usd": 0.009976 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_90c84e6db9bc", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1529, "completion_tokens": 125, "usd": 0.00012174 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3978, "completion_tokens": 478, "usd": 0.0016236 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3938, "completion_tokens": 1578, "usd": 0.008672 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_9f99d91e21f6", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1301, "completion_tokens": 185, "usd": 0.00012246 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_29cce0ac5de3", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1441, "completion_tokens": 103, "usd": 0.00011118 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_21bf7c25b237", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1442, "completion_tokens": 142, "usd": 0.0001206 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_256b30ee0efe", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1303, "completion_tokens": 147, "usd": 0.00011346 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_dfea8f3d0de4", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1272, "completion_tokens": 153, "usd": 0.00011304 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_f1ddb4f3653c", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1497, "completion_tokens": 126, "usd": 0.00012006 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_ef0acc9e5a05", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1459, "completion_tokens": 158, "usd": 0.00012546 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_11286bfe394d", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1185, "completion_tokens": 153, "usd": 0.00010782 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_325c7d095b64", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1256, "completion_tokens": 214, "usd": 0.00012672 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_2a529632a34a", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 148, "usd": 0.00011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_908da659aa18", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1203, "completion_tokens": 247, "usd": 0.00013146 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_3a7988db6226", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 214, "usd": 0.0001284 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_3186365c17c2", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2884, "completion_tokens": 539, "usd": 0.0003024 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2958, "completion_tokens": 302, "usd": 0.0011592 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_780ead4737e0", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 180, "usd": 0.00011742 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2457, "completion_tokens": 265, "usd": 0.0009756 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_0ba4c66e04ad", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 201, "usd": 0.0001263 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_58c7d9abe244", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1237, "completion_tokens": 193, "usd": 0.00012054 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2812, "completion_tokens": 334, "usd": 0.0011442 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2801, "completion_tokens": 4752, "usd": 0.017057 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_8e9bcc22b033", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 108, "usd": 0.0000942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_1cc2a5f894e5", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 80, "usd": 0.00008868 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1798, "completion_tokens": 219, "usd": 0.0007365 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1682, "completion_tokens": 1008, "usd": 0.004706 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_07aa4d542d5a", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 120, "usd": 0.00010074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00272886, "super": 0.0069507, "ultra": 0.040411 }, "counterfactual_ultra_usd": 0.105228, "spent_usd_by_tier": { "nano": 0.00272886, "super": 0.0069507, "ultra": 0.040959 }, "prompt_tokens_by_tier": { "nano": 30149, "super": 17427, "ultra": 12132 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 24.96, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r2-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R2", "expected_decision": "BLOCK", "edit_id": "edit_d879fb52b47e", "verdict_id": "verdict_d879fb52b47e", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The system replayed 13 test pairs for the fx-expense-converter skill after a documentation change. Twelve pairs passed with no regressions, drift, or improvements detected. One pair, \"Convert at a weekend date,\" was marked uncertain because the replay output showed a rate date of 2026-10-09 instead of the expected 2026-10-11, and Tavily detected the rate value shifting from 1.0768 to 1.0800. The run cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_88832e95c9b9", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1312, "completion_tokens": 110, "usd": 0.00010512 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_33a9c6adf157", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The replay fails the 'uses the correct date 2026-10-11' criterion by showing 2026-10-09 in the rate line (m003). Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1316, "completion_tokens": 170, "usd": 0.00011976 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2685, "completion_tokens": 297, "usd": 0.0010728 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2723, "completion_tokens": 2728, "usd": 0.010907 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_07551b4c2872", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1292, "completion_tokens": 196, "usd": 0.00012456 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f725843c1f80", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1394, "completion_tokens": 89, "usd": 0.000105 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_0545f282df0a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 228, "usd": 0.00013758 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_9df69746323e", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 98, "usd": 0.00010656 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_51bbbd84f8be", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1428, "completion_tokens": 138, "usd": 0.0001188 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_32ea0f012a38", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1343, "completion_tokens": 159, "usd": 0.00011874 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_1847ee62554a", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 126, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_08c9424f7da6", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1412, "completion_tokens": 160, "usd": 0.00012312 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_a144e4814df4", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 97, "usd": 0.00009156 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e625a12fe4a6", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 72, "usd": 0.00008748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1673, "completion_tokens": 223, "usd": 0.0007026 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1547, "completion_tokens": 1033, "usd": 0.004646 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c818c2acd85b", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1188, "completion_tokens": 137, "usd": 0.00010416 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00145908, "super": 0.0017754, "ultra": 0.015553 }, "counterfactual_ultra_usd": 0.044009, "spent_usd_by_tier": { "nano": 0.00150936, "super": 0.0019653, "ultra": 0.016348 }, "prompt_tokens_by_tier": { "nano": 17900, "super": 4946, "ultra": 4615 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 14.565, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_55c551c96357", "verdict_id": "verdict_55c551c96357", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: fx-expense-converter still handles all 13 past tasks.", "narrative": "Passed: fx-expense-converter still handles all 13 past tasks. The single-line SKILL.md change was evaluated against the full regression suite of 13 pairs with zero regressions, drift, or uncertain results. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_e9f830fa642e", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1346, "completion_tokens": 180, "usd": 0.00012396 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_bbae78eada7e", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1350, "completion_tokens": 139, "usd": 0.00011436 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_518252b4621f", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1326, "completion_tokens": 195, "usd": 0.00012636 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2718, "completion_tokens": 300, "usd": 0.0010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_6bd370a9ae4d", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1392, "completion_tokens": 100, "usd": 0.00010752 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_4a19b9d5037d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1391, "completion_tokens": 138, "usd": 0.00011658 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2718, "completion_tokens": 345, "usd": 0.0011259 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_659d9e0a0119", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 176, "usd": 0.0001266 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_ae6602b84c3d", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1435, "completion_tokens": 160, "usd": 0.0001245 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_76bd7f61f20c", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1377, "completion_tokens": 178, "usd": 0.00012534 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_7b5ee5b9fe63", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1402, "completion_tokens": 118, "usd": 0.00011244 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_40cfb64bd6bb", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1420, "completion_tokens": 133, "usd": 0.00011712 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2c596c557862", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 112, "usd": 0.00009516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_dd37987a313e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 68, "usd": 0.00008508 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1678, "completion_tokens": 217, "usd": 0.0006987 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1599, "completion_tokens": 1199, "usd": 0.005196 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_14951b257b04", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 98, "usd": 0.0000933 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00146832, "super": 0.00291, "ultra": 0.005196 }, "counterfactual_ultra_usd": 0.037573, "spent_usd_by_tier": { "nano": 0.00146832, "super": 0.00291, "ultra": 0.005628 }, "prompt_tokens_by_tier": { "nano": 17292, "super": 7114, "ultra": 1851 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 10.038, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-github-triage-2", "skill": "github-triage", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_edf1321f95ba", "verdict_id": "verdict_edf1321f95ba", "state": "done", "decision": "BLOCK", "pipeline_decision": "BLOCK", "action": "reverted", "reasons": [ "agreed_regression" ], "headline": "Blocked: github-triage now omits the repository name in the format `acme/` on 1 of 13 past tasks.", "narrative": "Blocked: github-triage now omits the repository name in the format `acme/` on 1 of 13 past tasks. The system ran 13 regression pairs against the previous version and found one failure where the skill stopped including the required repository prefix during triage. The other 12 tasks passed without drift or improvement. Nothing to do.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_0e10ad5bac34", "task_id": "tri-no-repo", "outcome": "regression", "expected_outcome": "regression", "decided_by": "ultra", "reason": "states the repository name in the format `acme/` is no longer met.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 149, "usd": 0.00010548 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1559, "completion_tokens": 241, "usd": 0.0006846 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1560, "completion_tokens": 1657, "usd": 0.006531 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_e27ff37067f7", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1217, "completion_tokens": 166, "usd": 0.00011286 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_3d9dd364ab27", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1360, "completion_tokens": 175, "usd": 0.0001236 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_570420a9b31d", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1373, "completion_tokens": 108, "usd": 0.0001083 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_40e6b8dc80bd", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 124, "usd": 0.000108 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_5a38101f8f14", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1354, "completion_tokens": 164, "usd": 0.0001206 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.9, "prompt_tokens": 3087, "completion_tokens": 299, "usd": 0.0011952 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_1f166314792c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1423, "completion_tokens": 268, "usd": 0.0001497 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_d9f90397db42", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1305, "completion_tokens": 235, "usd": 0.0001347 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_13d086a4d0a2", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1385, "completion_tokens": 199, "usd": 0.00013086 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_24cc0a76ffb7", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1380, "completion_tokens": 147, "usd": 0.00011808 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_86211c1cba50", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1162, "completion_tokens": 109, "usd": 0.00009588 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_70087be29148", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1559, "completion_tokens": 219, "usd": 0.0006648 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1471, "completion_tokens": 1524, "usd": 0.006043 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3b107f1aecf5", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1187, "completion_tokens": 104, "usd": 0.00009618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00149028, "super": 0.0025446, "ultra": 0.012574 }, "counterfactual_ultra_usd": 0.043874, "spent_usd_by_tier": { "nano": 0.00149028, "super": 0.0025446, "ultra": 0.013103 }, "prompt_tokens_by_tier": { "nano": 16758, "super": 6205, "ultra": 3326 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 10.577, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r6-github-triage-2", "skill": "github-triage", "type": "R6", "expected_decision": "BLOCK", "edit_id": "edit_26dbca6b252e", "verdict_id": "verdict_26dbca6b252e", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The edit changed the github-triage skill. Twelve replays passed and one was uncertain. The uncertain replay for \"Triage acme/infra-terraform\" failed to label issue #75 as documentation, instead labeling it question, which violates a success criterion. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_9aae4e6186d2", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1151, "completion_tokens": 117, "usd": 0.00009714 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_852cf7299d83", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1181, "completion_tokens": 127, "usd": 0.00010134 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_1f4cd7269ecb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1384, "completion_tokens": 155, "usd": 0.00012024 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d6db457362fe", "task_id": "tri-infra-terraform", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails to label issue #75 as documentation (labels it question instead), violating a success criterion. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 104, "usd": 0.00010932 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3651, "completion_tokens": 314, "usd": 0.0013779 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3575, "completion_tokens": 2712, "usd": 0.011711 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": true, "level0_new": 2, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_7d366fbfea40", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1319, "completion_tokens": 169, "usd": 0.0001197 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_bb40249ba200", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1334, "completion_tokens": 117, "usd": 0.00010812 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3082, "completion_tokens": 324, "usd": 0.0012162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_5cb5a86b5b0c", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1381, "completion_tokens": 214, "usd": 0.00013422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_8e097e2bc153", "task_id": "tri-mobile-app", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1341, "completion_tokens": 99, "usd": 0.00010422 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_dc72a3a720da", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1366, "completion_tokens": 175, "usd": 0.00012396 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_d4e29e63139d", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1392, "completion_tokens": 169, "usd": 0.00012408 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_1bcf329ed33b", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 172, "usd": 0.000111 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_20de0884c5fb", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 178, "usd": 0.0001122 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1558, "completion_tokens": 219, "usd": 0.0006645 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1539, "completion_tokens": 1472, "usd": 0.005955 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b123eb9839bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1199, "completion_tokens": 102, "usd": 0.00009642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00146196, "super": 0.0032586, "ultra": 0.017666 }, "counterfactual_ultra_usd": 0.050996, "spent_usd_by_tier": { "nano": 0.00146196, "super": 0.0032586, "ultra": 0.018237 }, "prompt_tokens_by_tier": { "nano": 16774, "super": 8291, "ultra": 5412 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 15.142, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_17daba64f297", "verdict_id": "verdict_17daba64f297", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The change only adds a documentation comment with no functional code modifications. Twelve test cases passed and zero regressions or drifts were detected. One case, \"Convert at a weekend date,\" was marked uncertain because Tavily detected a rate shift from 1.0768 to 1.0800 despite the skill diff containing no logic changes. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_4e739b0fed3d", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1324, "completion_tokens": 135, "usd": 0.00011184 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_d7ede738861c", "task_id": "fx-weekend-date", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a documentation comment and changes no functional steps, tools, arguments, or rules. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 144, "usd": 0.00011496 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2569, "completion_tokens": 283, "usd": 0.0010254 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2485, "completion_tokens": 1608, "usd": 0.007309 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": false, "old_value": "1.0768", "new_value": "1.0800" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_7f573055a9ef", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1316, "completion_tokens": 164, "usd": 0.00011832 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_f4c1c98c1a21", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1382, "completion_tokens": 140, "usd": 0.00011652 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_6b7462d13be4", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1381, "completion_tokens": 187, "usd": 0.00012774 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_914ca7824ded", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1384, "completion_tokens": 92, "usd": 0.00010512 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_da9dc2ce34c7", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1416, "completion_tokens": 135, "usd": 0.00011736 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_70ade45e8144", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1343, "completion_tokens": 200, "usd": 0.00012858 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_0b662be246d7", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 118, "usd": 0.00011472 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_ca8b25ae83a2", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1400, "completion_tokens": 163, "usd": 0.00012312 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_0bda149accdc", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 100, "usd": 0.000093 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e0d76dcf56dd", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 68, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1522, "completion_tokens": 221, "usd": 0.0006555 }, { "tier": "ultra", "verdict": "improved", "confidence": 0.95, "prompt_tokens": 1347, "completion_tokens": 1699, "usd": 0.006444 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_f3b045e8f40e", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 119, "usd": 0.00009978 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00145686, "super": 0.0016809, "ultra": 0.013753 }, "counterfactual_ultra_usd": 0.041872, "spent_usd_by_tier": { "nano": 0.0015069, "super": 0.0018708, "ultra": 0.01438 }, "prompt_tokens_by_tier": { "nano": 17923, "super": 4679, "ultra": 4153 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 13.172, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r1-github-triage-1", "skill": "github-triage", "type": "R1", "expected_decision": "BLOCK", "edit_id": "edit_bb05158b70d3", "verdict_id": "verdict_bb05158b70d3", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 4 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 4 of 13 replays. Review before merging. The system replayed 13 test cases against the proposed change to github-triage. Nine passed, but four were marked uncertain because the skill diff did not explain the output differences, suggesting possible nondeterminism. The affected cases include triage tasks for acme/status-page, acme/search-indexer, acme/mobile-app, and a math computation. No regressions or drifts were detected. Cost was negligible across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_fc1895f58dcf", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1127, "completion_tokens": 111, "usd": 0.00009426 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_99daafbd39ef", "task_id": "tri-status-page", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay output includes extra lines '#8: bug' and '#11: enhancement' not present in the golden output, violating the success criterion about additional text. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1240, "completion_tokens": 133, "usd": 0.00010632 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2658, "completion_tokens": 268, "usd": 0.0010386 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2593, "completion_tokens": 2117, "usd": 0.008944 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_204e00dc72de", "task_id": "tri-search-indexer", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1341, "completion_tokens": 81, "usd": 0.0000999 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3296, "completion_tokens": 312, "usd": 0.0012696 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3198, "completion_tokens": 3589, "usd": 0.013965 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_d8383ef71ebb", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1368, "completion_tokens": 189, "usd": 0.00012744 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_f644709d32f1", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1304, "completion_tokens": 135, "usd": 0.00011064 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_92e4b2c50b1c", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1321, "completion_tokens": 131, "usd": 0.0001107 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3306, "completion_tokens": 302, "usd": 0.0012636 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_574e79e6ac2e", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1404, "completion_tokens": 168, "usd": 0.00012456 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_7213bfe2b9f8", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1331, "completion_tokens": 187, "usd": 0.00012474 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3072, "completion_tokens": 312, "usd": 0.0012024 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 3080, "completion_tokens": 4998, "usd": 0.018074 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_a05f5333a824", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1401, "completion_tokens": 171, "usd": 0.0001251 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_1891e80878db", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1400, "completion_tokens": 162, "usd": 0.00012288 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d5662e056952", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 108, "usd": 0.0000942 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_e3d06ab590fa", "task_id": "canary-percent", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1605, "completion_tokens": 214, "usd": 0.0006741 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1467, "completion_tokens": 1922, "usd": 0.007233 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_18d4cf624d2d", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 105, "usd": 0.00009498 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00142248, "super": 0.0054483, "ultra": 0.048216 }, "counterfactual_ultra_usd": 0.088332, "spent_usd_by_tier": { "nano": 0.00142248, "super": 0.0054483, "ultra": 0.049012 }, "prompt_tokens_by_tier": { "nano": 16696, "super": 13937, "ultra": 10771 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 22.642, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r3-expense-report-2", "skill": "expense-report", "type": "R3", "expected_decision": "BLOCK", "edit_id": "edit_01fb05bb4ccc", "verdict_id": "verdict_01fb05bb4ccc", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 3 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 3 of 22 replays. Review before merging. The system ran 22 test pairs against the expense-report skill change; 19 passed but three were marked uncertain. One replay failed to extract required fields from a bakery receipt and missed a second receipt entirely. A second replay showed unexplained differences likely caused by nondeterminism. A third replay changed a parameter unrelated to the CSV cleaning task under test. Cost was minimal across all model tiers. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2f14f21d6d9f", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1106, "completion_tokens": 123, "usd": 0.00009588 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_e92a7282295d", "task_id": "exp-missing-receipt", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The replay fails all success criteria: it does not state merchant, date, total, VAT for the filed receipt (m004), does not state the second receipt was not found (m004), does not state grand total (m004), does not state number of receipts f…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1406, "completion_tokens": 138, "usd": 0.00011748 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3080, "completion_tokens": 324, "usd": 0.0012156 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3058, "completion_tokens": 4539, "usd": 0.016675 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_a6b930ea6aff", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1606, "completion_tokens": 134, "usd": 0.00012852 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3997, "completion_tokens": 374, "usd": 0.0015357 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3917, "completion_tokens": 2450, "usd": 0.011267 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_e2914d406490", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1289, "completion_tokens": 130, "usd": 0.00010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_07a85852522a", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 194, "usd": 0.0001323 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_85a67c3ff72c", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1454, "completion_tokens": 190, "usd": 0.00013284 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_afcfe0768c5a", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1291, "completion_tokens": 192, "usd": 0.00012354 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_a64afbf6b2cb", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1272, "completion_tokens": 106, "usd": 0.00010176 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_9c6ec62f2eb9", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1521, "completion_tokens": 153, "usd": 0.00012798 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_6ad0a889dbc3", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1471, "completion_tokens": 157, "usd": 0.00012594 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_b6e697e319df", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 148, "usd": 0.0001095 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_8ffffbf26485", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1256, "completion_tokens": 170, "usd": 0.00011616 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2885, "completion_tokens": 299, "usd": 0.0011346 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_56a48c4930ba", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 165, "usd": 0.00011358 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_4c8e63150ce8", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 223, "usd": 0.00012912 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2672, "completion_tokens": 296, "usd": 0.001068 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_aa8e42f0327e", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1373, "completion_tokens": 146, "usd": 0.00011742 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3375, "completion_tokens": 330, "usd": 0.0013095 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.05, "prompt_tokens": 3271, "completion_tokens": 2033, "usd": 0.00937 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_c3600f8c1037", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 234, "usd": 0.00013176 }, { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 2841, "completion_tokens": 69, "usd": 0.0009144 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2801, "completion_tokens": 2427, "usd": 0.010082 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_64e31b08b933", "task_id": "csv-header-only", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff changes the expense-report skill's receipts.parse parameter from 'file' to 'filename', which is unrelated to the leads.csv cleaning task. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1279, "completion_tokens": 253, "usd": 0.00013746 }, { "tier": "super", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 2988, "completion_tokens": 68, "usd": 0.0009576 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2983, "completion_tokens": 1777, "usd": 0.008314 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 1, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_c2e995113d65", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 129, "usd": 0.0001083 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_18bb09c7d462", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1225, "completion_tokens": 163, "usd": 0.00011262 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2771, "completion_tokens": 344, "usd": 0.0011409 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2679, "completion_tokens": 5166, "usd": 0.018177 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_de7f8d3695ad", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 97, "usd": 0.00009156 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_40e8a755ae4f", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 82, "usd": 0.00008988 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1672, "completion_tokens": 274, "usd": 0.0007482 }, { "tier": "ultra", "verdict": "improved", "confidence": 0.95, "prompt_tokens": 1586, "completion_tokens": 1354, "usd": 0.005648 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_b58e85552abd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 120, "usd": 0.00010074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00255288, "super": 0.0100245, "ultra": 0.079533 }, "counterfactual_ultra_usd": 0.152049, "spent_usd_by_tier": { "nano": 0.00255288, "super": 0.0100245, "ultra": 0.080297 }, "prompt_tokens_by_tier": { "nano": 28760, "super": 26281, "ultra": 20723 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 45.992, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "r5-expense-report-1", "skill": "expense-report", "type": "R5", "expected_decision": "BLOCK", "edit_id": "edit_86079f5d9ca8", "verdict_id": "verdict_86079f5d9ca8", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 5 of 22 replays. Review before merging.", "narrative": "Held: the judges were not sure on 5 of 22 replays. Review before merging. The diff only rewords the skill description in SKILL.md without changing tools, arguments, or rules. Seventeen replays passed, but five were held because the judges could not confirm the cosmetic changes were harmless. One replay showed likely nondeterminism unrelated to the edit. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_2cfa611323e6", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1094, "completion_tokens": 154, "usd": 0.0001026 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_f3f61876a85b", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1444, "completion_tokens": 134, "usd": 0.0001188 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2794, "completion_tokens": 305, "usd": 0.0011127 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2786, "completion_tokens": 2814, "usd": 0.011228 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_03e985bf60c4", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1565, "completion_tokens": 101, "usd": 0.00011814 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3813, "completion_tokens": 414, "usd": 0.0015165 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3723, "completion_tokens": 2092, "usd": 0.009999 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_b14c6d6a1857", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1327, "completion_tokens": 139, "usd": 0.00011298 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2623, "completion_tokens": 297, "usd": 0.0010542 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_758cddadc695", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "regression", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1478, "completion_tokens": 146, "usd": 0.00012372 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5e47aca6907b", "task_id": "exp-dinner-parking", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description text in SKILL.md and does not change any tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 1397, "completion_tokens": 190, "usd": 0.00012942 }, { "tier": "super", "verdict": "regressed", "confidence": 0.94, "prompt_tokens": 3106, "completion_tokens": 303, "usd": 0.0012045 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3088, "completion_tokens": 2915, "usd": 0.011833 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a8a18a97157f", "task_id": "exp-office-supplies", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field (changing 'receipt' to 'document') without altering any logic, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1280, "completion_tokens": 131, "usd": 0.00010824 }, { "tier": "super", "verdict": "regressed", "confidence": 0.9, "prompt_tokens": 2500, "completion_tokens": 279, "usd": 0.0010011 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 2460, "completion_tokens": 1995, "usd": 0.008445 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_2a8fe0325d1e", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1320, "completion_tokens": 134, "usd": 0.00011136 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_21460568387a", "task_id": "exp-lunch-train", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only changes the description text in SKILL.md, broadening 'receipts' to 'receipts, invoices, or other financial documents' — a cosmetic reword that does not alter any tool, step, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1506, "completion_tokens": 143, "usd": 0.00012468 }, { "tier": "super", "verdict": "regressed", "confidence": 0.92, "prompt_tokens": 3286, "completion_tokens": 384, "usd": 0.0013314 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 3208, "completion_tokens": 2119, "usd": 0.009565 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_1b2e69a62019", "task_id": "exp-berlin-trip", "outcome": "uncertain", "expected_outcome": "regression", "decided_by": "ultra", "reason": "The skill diff only rewords the description field (receipts → receipts, invoices, or other financial documents; receipt file → file; receipt → document) without changing any tool, argument, step, or logic. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1442, "completion_tokens": 138, "usd": 0.00011964 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3104, "completion_tokens": 401, "usd": 0.0012921 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 3048, "completion_tokens": 2090, "usd": 0.009318 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_df0530c17b4e", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 146, "usd": 0.00010974 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_5b34a19cd981", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1235, "completion_tokens": 114, "usd": 0.00010146 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_b1ea18acb350", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 106, "usd": 0.00010014 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_6d6c20462d5a", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1251, "completion_tokens": 136, "usd": 0.0001077 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_1d5047ec8fd2", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1284, "completion_tokens": 101, "usd": 0.00010128 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2706, "completion_tokens": 280, "usd": 0.0010638 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2569, "completion_tokens": 3092, "usd": 0.011845 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_a7ca6fd2483c", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 143, "usd": 0.00010992 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2652, "completion_tokens": 317, "usd": 0.0010809 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_8ed25d9902cb", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "regression", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1213, "completion_tokens": 206, "usd": 0.00012222 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2195, "completion_tokens": 298, "usd": 0.0009267 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_bdb9bd3422e8", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1301, "completion_tokens": 180, "usd": 0.00012126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_03c42fb9fb74", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "regression", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 153, "usd": 0.00011166 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2610, "completion_tokens": 297, "usd": 0.0010503 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2558, "completion_tokens": 3021, "usd": 0.011621 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_d54e294c1ad8", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1162, "completion_tokens": 98, "usd": 0.00009324 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_3cb515f6830e", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1170, "completion_tokens": 116, "usd": 0.00009804 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_c51d156486d2", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1187, "completion_tokens": 105, "usd": 0.00009642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00244266, "super": 0.0126342, "ultra": 0.083854 }, "counterfactual_ultra_usd": 0.163665, "spent_usd_by_tier": { "nano": 0.00244266, "super": 0.0126342, "ultra": 0.084646 }, "prompt_tokens_by_tier": { "nano": 28655, "super": 31389, "ultra": 23974 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 32.372, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-expense-report-1", "skill": "expense-report", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_6d4c31c9e013", "verdict_id": "verdict_6d4c31c9e013", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear" ], "headline": "Passed: expense-report still handles all 22 past tasks.", "narrative": "Passed: expense-report still handles all 22 past tasks. The edit updated SKILL.md with two additions and zero deletions. The system re-ran the full regression suite of 22 pairs covering the skill's known behaviors. Every pair passed with zero regressions, zero drift, and zero uncertain results. Cost was negligible across all model tiers. Nothing to do.", "pairs": [ { "golden_run_id": "run_abb4c4a8ce34", "replay_run_id": "run_9797515450fd", "task_id": "exp-no-files", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1070, "completion_tokens": 127, "usd": 0.00009468 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_26e9cb4736f7", "replay_run_id": "run_562f1bacbb4a", "task_id": "exp-missing-receipt", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1405, "completion_tokens": 192, "usd": 0.00013038 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_2f27937bebcf", "replay_run_id": "run_118b690af2c2", "task_id": "exp-ink-fuel-email", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1622, "completion_tokens": 121, "usd": 0.00012636 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 3810, "completion_tokens": 414, "usd": 0.0015156 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 3758, "completion_tokens": 1754, "usd": 0.00902 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_67d9a9aaa598", "replay_run_id": "run_17c39cd8a28a", "task_id": "exp-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1313, "completion_tokens": 186, "usd": 0.00012342 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_aa56356b5787", "replay_run_id": "run_89a8beb01ad9", "task_id": "exp-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1429, "completion_tokens": 132, "usd": 0.00011742 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_dbff77bc2a71", "replay_run_id": "run_5a08a4d190cc", "task_id": "exp-dinner-parking", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1466, "completion_tokens": 148, "usd": 0.00012348 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_783db1ad7716", "replay_run_id": "run_a04984da2f2e", "task_id": "exp-office-supplies", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1315, "completion_tokens": 180, "usd": 0.0001221 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_f15819d8642f", "replay_run_id": "run_6c284a0936b6", "task_id": "exp-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1332, "completion_tokens": 108, "usd": 0.00010584 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_047f259274fb", "replay_run_id": "run_37dcfb667a2b", "task_id": "exp-lunch-train", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1533, "completion_tokens": 128, "usd": 0.0001227 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ccf986b513f7", "replay_run_id": "run_f63fa53e365f", "task_id": "exp-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1471, "completion_tokens": 158, "usd": 0.00012618 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_596660827ddb", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 143, "usd": 0.00010902 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_35dd5c46b4f3", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1259, "completion_tokens": 219, "usd": 0.0001281 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_24bcb89d6737", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1233, "completion_tokens": 151, "usd": 0.00011022 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_3053a2b19841", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1239, "completion_tokens": 143, "usd": 0.00010866 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_df65d3ad0120", "task_id": "csv-tickets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1272, "completion_tokens": 179, "usd": 0.00011928 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_175a9b556a3d", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2812, "completion_tokens": 495, "usd": 0.00028752 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2640, "completion_tokens": 300, "usd": 0.001062 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_6396f7029b97", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1312, "completion_tokens": 167, "usd": 0.0001188 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2430, "completion_tokens": 312, "usd": 0.0010098 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 1, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_7ff7643575cd", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 191, "usd": 0.00012318 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_07fbc9fb046d", "task_id": "csv-signups", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1249, "completion_tokens": 148, "usd": 0.00011046 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2572, "completion_tokens": 361, "usd": 0.0010965 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2592, "completion_tokens": 4631, "usd": 0.016485 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_11e6e85db7b9", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1150, "completion_tokens": 109, "usd": 0.00009516 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_6781c847f496", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1146, "completion_tokens": 72, "usd": 0.00008604 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1513, "completion_tokens": 219, "usd": 0.000651 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 1413, "completion_tokens": 1661, "usd": 0.006396 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_16136c6d8859", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 82, "usd": 0.00009162 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00268062, "super": 0.0053349, "ultra": 0.031901 }, "counterfactual_ultra_usd": 0.090782, "spent_usd_by_tier": { "nano": 0.00268062, "super": 0.0053349, "ultra": 0.032373 }, "prompt_tokens_by_tier": { "nano": 30361, "super": 12965, "ultra": 8004 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 25.136, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-csv-cleaner-1", "skill": "csv-cleaner", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_88ee23b9cfb1", "verdict_id": "verdict_88ee23b9cfb1", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 2 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 2 of 13 replays. Review before merging. The system replayed 13 test cases against the csv-cleaner skill change; 11 passed but 2 were held for human review. For \"Clean support/tickets.csv\" the skill diff did not explain the difference, suggesting nondeterminism. For \"Clean signups.csv\" all six success criteria were met by the replay file operations, yet the judge remained uncertain. Cost was negligible across all models. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_500bf90d565e", "replay_run_id": "run_95b0aeaae315", "task_id": "csv-header-only", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 153, "usd": 0.0001131 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 2115, "completion_tokens": 309, "usd": 0.0009126 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_fcbf5e62eb78", "replay_run_id": "run_ad0ef0f6f262", "task_id": "csv-already-clean", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1227, "completion_tokens": 153, "usd": 0.00011034 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_add0c7098f3f", "replay_run_id": "run_4bc22f69c2f4", "task_id": "csv-vendors", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1257, "completion_tokens": 130, "usd": 0.00010662 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8d70cc039e47", "replay_run_id": "run_341f1aafe06d", "task_id": "csv-tickets", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "the skill diff does not explain the difference; likely nondeterminism. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1236, "completion_tokens": 101, "usd": 0.0000984 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2681, "completion_tokens": 278, "usd": 0.0010545 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2572, "completion_tokens": 4318, "usd": 0.015526 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_88ff5a1498e1", "replay_run_id": "run_89cbeb5f1106", "task_id": "csv-payments", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1260, "completion_tokens": 146, "usd": 0.00011064 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 2609, "completion_tokens": 286, "usd": 0.0010401 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_77f5ca49aaa1", "replay_run_id": "run_638b93eff172", "task_id": "csv-attendees", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1289, "completion_tokens": 135, "usd": 0.00010974 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4f154f4c7a37", "replay_run_id": "run_57f1a1c928ec", "task_id": "csv-inventory", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1247, "completion_tokens": 153, "usd": 0.00011154 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0978882c5608", "replay_run_id": "run_ed700d3411da", "task_id": "csv-signups", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "All six success criteria are met by the replay's actual file operations: signups.clean.csv has 3 rows with no empty IDs or duplicates (t003), signups.csv has 4 data rows with 1 empty ID (t001, t002), and the clean file is created from the o…. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1273, "completion_tokens": 122, "usd": 0.00010566 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2507, "completion_tokens": 305, "usd": 0.0010266 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.7, "prompt_tokens": 2464, "completion_tokens": 2857, "usd": 0.011035 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_02d06d2ccc47", "replay_run_id": "run_f83b9f46b05b", "task_id": "csv-orders", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1271, "completion_tokens": 173, "usd": 0.00011778 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a5e219ae9b07", "replay_run_id": "run_7323ba2c2619", "task_id": "csv-customers", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1245, "completion_tokens": 139, "usd": 0.00010806 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_2e08956d5846", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 81, "usd": 0.00008988 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_8ad7f2f013aa", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 72, "usd": 0.00008676 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1517, "completion_tokens": 219, "usd": 0.0006522 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1399, "completion_tokens": 1828, "usd": 0.006883 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_3f0ad4c092bd", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1163, "completion_tokens": 129, "usd": 0.00010074 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00136926, "super": 0.004686, "ultra": 0.033444 }, "counterfactual_ultra_usd": 0.070198, "spent_usd_by_tier": { "nano": 0.00136926, "super": 0.004686, "ultra": 0.034172 }, "prompt_tokens_by_tier": { "nano": 16073, "super": 11429, "ultra": 6815 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 19.076, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "n0-github-triage-1", "skill": "github-triage", "type": "N0", "expected_decision": "PASS", "edit_id": "edit_21b2cd0ac196", "verdict_id": "verdict_21b2cd0ac196", "state": "done", "decision": "WARN", "pipeline_decision": "WARN", "action": "held", "reasons": [ "uncertain" ], "headline": "Held: the judges were not sure on 1 of 13 replays. Review before merging.", "narrative": "Held: the judges were not sure on 1 of 13 replays. Review before merging. The change adds only a comment line to the skill file with no logic modifications. Twelve test pairs passed and one replay for \"Triage acme/mobile-app\" was marked uncertain because the diff does not alter any tool, argument, or rule. The judges held the decision for human review. Approve if this is intended.", "pairs": [ { "golden_run_id": "run_5af7fa46de07", "replay_run_id": "run_c6a8e1abb734", "task_id": "tri-no-repo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1163, "completion_tokens": 99, "usd": 0.00009354 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a192f2b86480", "replay_run_id": "run_4e23436145b0", "task_id": "tri-status-page", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1205, "completion_tokens": 137, "usd": 0.00010518 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3cf2a3950c01", "replay_run_id": "run_e1dd839734fb", "task_id": "tri-search-indexer", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1384, "completion_tokens": 151, "usd": 0.00011928 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_3c290a817d6a", "replay_run_id": "run_860252f0aeb1", "task_id": "tri-infra-terraform", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.96, "prompt_tokens": 1380, "completion_tokens": 137, "usd": 0.00011568 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_8c17017072f1", "replay_run_id": "run_ac0076e8ac87", "task_id": "tri-auth-service", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1307, "completion_tokens": 120, "usd": 0.00010722 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ce7061d73a4f", "replay_run_id": "run_0247b5972b80", "task_id": "tri-data-pipeline", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.88, "prompt_tokens": 1334, "completion_tokens": 109, "usd": 0.0001062 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3031, "completion_tokens": 318, "usd": 0.0011955 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 3, "level0_changed": 0 }, { "golden_run_id": "run_9660f7c09639", "replay_run_id": "run_07bb9b326000", "task_id": "tri-docs-site", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1478, "completion_tokens": 153, "usd": 0.0001254 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_074ba25642f7", "replay_run_id": "run_ef274337a616", "task_id": "tri-mobile-app", "outcome": "uncertain", "expected_outcome": "pass", "decided_by": "ultra", "reason": "The skill diff only adds a comment line '' which does not change any logic, tool, argument, or rule. Held for a human.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1340, "completion_tokens": 130, "usd": 0.0001116 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2982, "completion_tokens": 298, "usd": 0.0011628 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2948, "completion_tokens": 2608, "usd": 0.010772 } ], "control_outcome": "agrees", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_6e0454ff42b0", "replay_run_id": "run_ee6719208401", "task_id": "tri-billing-api", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1409, "completion_tokens": 146, "usd": 0.00011958 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ac4951646a0e", "replay_run_id": "run_44c02d5b3570", "task_id": "tri-widgets", "outcome": "pass", "expected_outcome": "pass", "decided_by": "super", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1389, "completion_tokens": 156, "usd": 0.00012078 }, { "tier": "super", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 3125, "completion_tokens": 306, "usd": 0.0012129 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_bfdc8aa6fc39", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 97, "usd": 0.00009156 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_a63316f74650", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "pass", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1134, "completion_tokens": 135, "usd": 0.00010044 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1515, "completion_tokens": 220, "usd": 0.0006525 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.5, "prompt_tokens": 1440, "completion_tokens": 1097, "usd": 0.004731 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_86b52a6672e1", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1175, "completion_tokens": 106, "usd": 0.00009594 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.0014124, "super": 0.0042237, "ultra": 0.015503 }, "counterfactual_ultra_usd": 0.051446, "spent_usd_by_tier": { "nano": 0.0014124, "super": 0.0042237, "ultra": 0.016081 }, "prompt_tokens_by_tier": { "nano": 16836, "super": 10653, "ultra": 4699 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 12.558, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-2", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_24224b18bee7", "verdict_id": "verdict_24224b18bee7", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 1 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 1 changed because the world did, not the skill. The skill ran against 13 test pairs with 12 passing and 1 drift. The drift occurred in \"Convert London costs from GBP\" where Tavily detected the GBP/USD exchange rate moved from 1.2702 to 1.3105 on 2026-10-13, causing the output to update correctly. No regressions or uncertainties were found. The skill logic remains sound; only the external exchange rate data changed. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_3e8797c16149", "task_id": "fx-no-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1336, "completion_tokens": 168, "usd": 0.00012048 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_608d63d15c36", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 165, "usd": 0.00012072 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_229dc213dbd6", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 165, "usd": 0.00011784 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_e71800e8113a", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1406, "completion_tokens": 93, "usd": 0.00010668 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_16ce4eaa1d1a", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 110, "usd": 0.00010854 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_bd178338e4b1", "task_id": "fx-london", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The GBP/USD exchange rate on 2026-10-13 was 1.2702 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1396, "completion_tokens": 189, "usd": 0.00012912 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2585, "completion_tokens": 344, "usd": 0.0010851 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2582, "completion_tokens": 1344, "usd": 0.006614 } ], "control_outcome": null, "drift": [ { "fact_id": "t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.2702", "new_value": "1.3105" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_f6108436cd3f", "task_id": "fx-coworking-courier", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1451, "completion_tokens": 134, "usd": 0.00011922 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_133f03cc5707", "task_id": "fx-software", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1355, "completion_tokens": 190, "usd": 0.0001269 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_cb6d6a6fa18e", "task_id": "fx-hamburg-conference", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1440, "completion_tokens": 126, "usd": 0.00011664 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_9bc309eb22db", "task_id": "fx-berlin-trip", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1377, "completion_tokens": 157, "usd": 0.0001203 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_c0d90d4cb42f", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1138, "completion_tokens": 97, "usd": 0.00009156 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_7d778bb3b477", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 168, "usd": 0.0001098 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1563, "completion_tokens": 221, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.85, "prompt_tokens": 1461, "completion_tokens": 1489, "usd": 0.005928 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_9620dc2201ff", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 102, "usd": 0.00009642 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00148422, "super": 0.0017529, "ultra": 0.012542 }, "counterfactual_ultra_usd": 0.041258, "spent_usd_by_tier": { "nano": 0.0015345, "super": 0.0019494, "ultra": 0.013268 }, "prompt_tokens_by_tier": { "nano": 17983, "super": 4752, "ultra": 4379 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 20.623, "simulated_models": false, "repeat": 0, "error": null }, { "bench_id": "d1-fx-expense-converter-1", "skill": "fx-expense-converter", "type": "D1", "expected_decision": "PASS", "edit_id": "edit_65a1fcfdc692", "verdict_id": "verdict_65a1fcfdc692", "state": "done", "decision": "PASS", "pipeline_decision": "PASS", "action": "merged", "reasons": [ "all_clear", "drift" ], "headline": "Passed: 13 tasks checked; 5 changed because the world did, not the skill.", "narrative": "Passed: 13 tasks checked; 5 changed because the world did, not the skill. The evaluation ran 13 test cases against the updated skill documentation. Eight cases passed with identical outputs. Five cases showed drift because the EUR/USD exchange rate for 2026-10-14 moved from 1.0800 to 1.163, and the skill correctly applied the new rate. No regressions or improvements were detected. The skill logic is sound; the variance reflects accurate external data retrieval. Re-bless the changed tasks.", "pairs": [ { "golden_run_id": "run_54f6942c5722", "replay_run_id": "run_24416e3f8916", "task_id": "fx-no-date", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1348, "completion_tokens": 198, "usd": 0.0001284 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2367, "completion_tokens": 395, "usd": 0.0010656 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2368, "completion_tokens": 1468, "usd": 0.006772 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_c2da5b1dcaba", "replay_run_id": "run_c04a94e117fd", "task_id": "fx-weekend-date", "outcome": "pass", "expected_outcome": "drift", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1352, "completion_tokens": 169, "usd": 0.00012168 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_ad35067adb80", "replay_run_id": "run_823b4ddae43a", "task_id": "fx-tokyo", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1304, "completion_tokens": 162, "usd": 0.00011712 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_b1d7632c70a9", "replay_run_id": "run_9326a185dca9", "task_id": "fx-paris", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1402, "completion_tokens": 98, "usd": 0.00010764 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_bc19570294da", "replay_run_id": "run_48d13ccbd36d", "task_id": "fx-zurich", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1369, "completion_tokens": 232, "usd": 0.00013782 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_56d3567fcf6b", "replay_run_id": "run_24d29ab3083c", "task_id": "fx-london", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1407, "completion_tokens": 183, "usd": 0.00012834 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_0eca9b474c8d", "replay_run_id": "run_d540794f1c98", "task_id": "fx-coworking-courier", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1439, "completion_tokens": 187, "usd": 0.00013122 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2572, "completion_tokens": 377, "usd": 0.0011109 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2613, "completion_tokens": 1841, "usd": 0.008136 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_710ce8599690", "replay_run_id": "run_ecc9e77802a1", "task_id": "fx-software", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "finance.rate returned 1.08 on 2026-10-14 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1367, "completion_tokens": 211, "usd": 0.00013266 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2375, "completion_tokens": 356, "usd": 0.0010329 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.9, "prompt_tokens": 2455, "completion_tokens": 1061, "usd": 0.005638 } ], "control_outcome": null, "drift": [ { "fact_id": "obs_t001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.08", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_e4953482fa67", "replay_run_id": "run_ce1c4a74b4df", "task_id": "fx-hamburg-conference", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1452, "completion_tokens": 182, "usd": 0.0001308 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2863, "completion_tokens": 317, "usd": 0.0011442 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2862, "completion_tokens": 1165, "usd": 0.006357 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_ed86b01db723", "replay_run_id": "run_84352baa167a", "task_id": "fx-berlin-trip", "outcome": "drift", "expected_outcome": "drift", "decided_by": "drift", "reason": "The EUR/USD exchange rate on 2026-10-14 was 1.0800 changed since the original run; the new output follows from it.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1412, "completion_tokens": 165, "usd": 0.00012432 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 2567, "completion_tokens": 331, "usd": 0.001068 }, { "tier": "ultra", "verdict": "uncertain", "confidence": 0.95, "prompt_tokens": 2600, "completion_tokens": 1388, "usd": 0.006764 } ], "control_outcome": null, "drift": [ { "fact_id": "f001", "changed": true, "status": "checked", "attributable": true, "old_value": "1.0800", "new_value": "1.163" } ], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 1 }, { "golden_run_id": "run_4e70e4f1af8e", "replay_run_id": "run_68a265f4d0b3", "task_id": "canary-formal-rewrite", "outcome": "pass", "expected_outcome": "drift", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.95, "prompt_tokens": 1174, "completion_tokens": 111, "usd": 0.00009708 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_a3387b112792", "replay_run_id": "run_eac0515b4a0c", "task_id": "canary-percent", "outcome": "pass", "expected_outcome": "drift", "decided_by": "control", "reason": "The unchanged skill also produced a different result on this task, so the difference is nondeterminism, not the edit.", "opinions": [ { "tier": "nano", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1158, "completion_tokens": 68, "usd": 0.0000858 }, { "tier": "super", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1563, "completion_tokens": 221, "usd": 0.0006678 }, { "tier": "ultra", "verdict": "regressed", "confidence": 0.95, "prompt_tokens": 1412, "completion_tokens": 1367, "usd": 0.005513 } ], "control_outcome": "diverges", "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 }, { "golden_run_id": "run_e1c459d18318", "replay_run_id": "run_aa0845d7fbd7", "task_id": "canary-summary", "outcome": "pass", "expected_outcome": "pass", "decided_by": "nano", "reason": "Output meets the criteria; differences are cosmetic.", "opinions": [ { "tier": "nano", "verdict": "equivalent", "confidence": 0.99, "prompt_tokens": 1199, "completion_tokens": 104, "usd": 0.0000969 } ], "control_outcome": null, "drift": [], "level0_hard_fail": false, "level0_new": 0, "level0_missing": 0, "level0_changed": 0 } ], "judge_usd_by_tier": { "nano": 0.00153978, "super": 0.0060894, "ultra": 0.03918 }, "counterfactual_ultra_usd": 0.083071, "spent_usd_by_tier": { "nano": 0.0020652, "super": 0.0070368, "ultra": 0.040181 }, "prompt_tokens_by_tier": { "nano": 23076, "super": 17210, "ultra": 14948 }, "cached_prompt_tokens_by_tier": { "nano": 0, "super": 0, "ultra": 0 }, "latency_s": 23.116, "simulated_models": false, "repeat": 0, "error": null } ] } } }