{ "schema_version": 1, "generated_at": "2026-09-21T06:27:47.369104+00:00", "status": "complete", "publication_ready": true, "publication_gate": "Passed: all five streams complete; native replay, hosted aggregate replay, and hosted raw-response replay audits agree with exact source hashes.", "required_streams": [ "2b", "9b", "jev", "luna", "astra" ], "title": "JevBench public subset: previously released Open-Jev baselines", "scope": "231 public JevBench tasks only: 72 original, 48 easy, 111 hard. The private and judge tiers are unavailable; this is not the full 534-task benchmark and no full-benchmark composite is computed. ZefanCai Open-Jev uses Qwen LoRA weights and a decision head; it is a different system from the Kotoba and Codiv projects also called OpenJev.", "benchmark": { "name": "JevBench public subset", "upstream_url": "https://github.com/fstandhartinger/jevbench", "upstream_commit": "f8ce71361165846101d02ebc83ad44e47ae44fc3", "execution_code_commit": "3c5bb4a5451ccfd81a5b9f2e05edaf2103805c94", "input_sha256": "2e20ff5f94dd04d015b3a7b9abd955757b6c5be25b55b97fcd141cb52f88b4aa", "dataset_sha256": "dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51", "public_tasks": 231, "full_benchmark_tasks": 534, "unavailable_tasks": 303, "public_tiers": { "original": 72, "easy": 48, "hard": 111 }, "task_type_inventory": { "noul": 74, "choice": 139, "score": 18 }, "full_benchmark_score": null }, "protocol": { "sum_tolerance_strict": 0.001, "sum_tolerance_rounding": 0.02, "rounding_policy": "Vectors outside strict tolerance but inside rounding tolerance are normalized and scored; vectors outside rounding tolerance are invalid and incorrect.", "argmax_tie_break": "lexicographically_smallest_label", "score_accuracy": "argmax_level_equals_gold_level", "ordinal_mae": "absolute_error_of_expected_level", "concurrency": 1, "warmups": 0, "retries": 0, "openjev": { "hardware": "NVIDIA H100 80GB HBM3", "device_count": 1, "model_order": [ "2b", "9b" ], "candidate_batch_size": 1, "max_length": 16384, "prefix_cache": false, "timing": "Loopback HTTP full-response wall time including validation; model loading outside timer." }, "hosted": { "timing": "Full HTTPS request wall time including response validation.", "gpt_output": "Verbalized probabilities in constrained JSON, not token logprobs.", "gpt_max_completion_tokens": 4096, "gpt_reasoning_effort": { "gpt-5.6-luna": "none", "gpt-6-astra": "low" }, "gpt_temperature": "omitted" }, "candidate_order": { "choice_tasks": 139, "different_native_vs_gpt_order": 119, "native": "Upstream criteria-map order preserved.", "gpt": "Upstream adapter task.labels order.", "order_controlled_comparison": false }, "latency_limit": "Heterogeneous quality-run diagnostics with one observation per task. No repeated latency benchmark, matched hardware, throughput, or energy-efficiency claim.", "cost_limit": "Hosted usage-times-tariff estimates are not billing receipts. Local compute is unmetered, not free.", "probability_limit": "Native and verbalized probability metrics do not measure equivalent intrinsic model confidence." }, "results": [ { "id": "2b", "label": "Released Open-Jev 2B", "status": "complete", "audit_status": "passed", "probability_source": "native", "release_status": "previously_released_checkpoint", "metrics": { "n_planned": 231, "n_attempted": 231, "n_scorable": 231, "n_valid": 231, "n_correct": 150, "accuracy": 0.6493506493506493, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 231, "brier_mean": 0.47505587229589324, "ordinal_mae": 0.362171166043865, "n_strict_valid": 231, "ece_top_label_10_bins": 0.12740565725687925, "latency": { "n": 231, "p50_ms": 138.00172321498394, "p95_ms": 205.81924449652433 } }, "per_public_tier": { "original": { "n_planned": 72, "n_attempted": 72, "n_scorable": 72, "n_valid": 72, "n_correct": 56, "accuracy": 0.7777777777777778, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 72, "brier_mean": 0.3255402072036015, "ordinal_mae": 0.17242147177743009, "n_strict_valid": 72, "ece_top_label_10_bins": 0.05817091540077076, "latency": { "n": 72, "p50_ms": 132.65516888350248, "p95_ms": 202.48594144359234 } }, "easy": { "n_planned": 48, "n_attempted": 48, "n_scorable": 48, "n_valid": 48, "n_correct": 48, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 48, "brier_mean": 0.026267081958640328, "ordinal_mae": null, "n_strict_valid": 48, "ece_top_label_10_bins": 0.09232735503995657, "latency": { "n": 48, "p50_ms": 163.80710806697607, "p95_ms": 170.6298009492457 } }, "hard": { "n_planned": 111, "n_attempted": 111, "n_scorable": 111, "n_valid": 111, "n_correct": 46, "accuracy": 0.4144144144144144, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 111, "brier_mean": 0.7661098346637594, "ordinal_mae": 0.7416705545767348, "n_strict_valid": 111, "ece_top_label_10_bins": 0.27732794330897187, "latency": { "n": 111, "p50_ms": 147.9222159832716, "p95_ms": 227.86736395210028 } } }, "per_family": { "adequacy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 8, "accuracy": 0.6666666666666666, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.5124827033130445, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.22777458569260003, "latency": { "n": 12, "p50_ms": 41.797625832259655, "p95_ms": 43.53315131738782 } }, "adversarial": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 3, "accuracy": 0.5, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.5878286141103769, "ordinal_mae": 0.5567559931232973, "n_strict_valid": 6, "ece_top_label_10_bins": 0.41209260624669736, "latency": { "n": 6, "p50_ms": 154.4183949008584, "p95_ms": 162.7946305088699 } }, "ambiguous": { "n_planned": 7, "n_attempted": 7, "n_scorable": 7, "n_valid": 7, "n_correct": 1, "accuracy": 0.14285714285714285, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 7, "brier_mean": 0.910323683054067, "ordinal_mae": null, "n_strict_valid": 7, "ece_top_label_10_bins": 0.596288245569952, "latency": { "n": 7, "p50_ms": 118.74120682477951, "p95_ms": 142.80385002493855 } }, "extraction": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 23, "accuracy": 0.9583333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 0.0968251175764338, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.17094051448457004, "latency": { "n": 24, "p50_ms": 132.8608077019453, "p95_ms": 166.96619540452957 } }, "fact": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0003589499255579256, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.010052850409974279, "latency": { "n": 12, "p50_ms": 34.705656580626965, "p95_ms": 39.539640583097935 } }, "intent": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 17, "accuracy": 0.7083333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 0.31592479719068806, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.20599748193511314, "latency": { "n": 24, "p50_ms": 166.53066221624613, "p95_ms": 196.5619483962655 } }, "judge_hard": { "n_planned": 17, "n_attempted": 17, "n_scorable": 17, "n_valid": 17, "n_correct": 9, "accuracy": 0.5294117647058824, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 17, "brier_mean": 0.6467380307143731, "ordinal_mae": null, "n_strict_valid": 17, "ece_top_label_10_bins": 0.3271456195779145, "latency": { "n": 17, "p50_ms": 41.42206907272339, "p95_ms": 44.280972704291344 } }, "long_policy": { "n_planned": 19, "n_attempted": 19, "n_scorable": 19, "n_valid": 19, "n_correct": 6, "accuracy": 0.3157894736842105, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 19, "brier_mean": 0.9691625207304174, "ordinal_mae": 0.6561374016056162, "n_strict_valid": 19, "ece_top_label_10_bins": 0.4843968464227859, "latency": { "n": 19, "p50_ms": 177.18532867729664, "p95_ms": 301.4282336458564 } }, "multi_hop": { "n_planned": 18, "n_attempted": 18, "n_scorable": 18, "n_valid": 18, "n_correct": 9, "accuracy": 0.5, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 18, "brier_mean": 0.6141568822158533, "ordinal_mae": 0.06741054057970075, "n_strict_valid": 18, "ece_top_label_10_bins": 0.14255194071208627, "latency": { "n": 18, "p50_ms": 163.44234347343445, "p95_ms": 233.01085457205772 } }, "ordinal": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.06281132466940156, "ordinal_mae": 0.17242147177743009, "n_strict_valid": 12, "ece_top_label_10_bins": 0.11614300779328485, "latency": { "n": 12, "p50_ms": 133.44534393399954, "p95_ms": 160.42679036036134 } }, "policy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 11, "accuracy": 0.9166666666666666, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.1745454627789654, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.15031208330175266, "latency": { "n": 12, "p50_ms": 43.86923648416996, "p95_ms": 327.91864871978714 } }, "probability": { "n_planned": 10, "n_attempted": 10, "n_scorable": 10, "n_valid": 10, "n_correct": 4, "accuracy": 0.4, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 10, "brier_mean": 0.9427854795610155, "ordinal_mae": null, "n_strict_valid": 10, "ece_top_label_10_bins": 0.422799565961523, "latency": { "n": 10, "p50_ms": 116.05329439043999, "p95_ms": 122.73405734449624 } }, "routing": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 9, "accuracy": 0.75, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.45214788388122923, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.2941758598147389, "latency": { "n": 12, "p50_ms": 197.39334471523762, "p95_ms": 204.14711330085993 } }, "routing_hard": { "n_planned": 5, "n_attempted": 5, "n_scorable": 5, "n_valid": 5, "n_correct": 4, "accuracy": 0.8, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 5, "brier_mean": 0.1629759267249182, "ordinal_mae": null, "n_strict_valid": 5, "ece_top_label_10_bins": 0.23556677597648737, "latency": { "n": 5, "p50_ms": 183.78658033907413, "p95_ms": 213.14927116036415 } }, "temporal_numeric": { "n_planned": 15, "n_attempted": 15, "n_scorable": 15, "n_valid": 15, "n_correct": 1, "accuracy": 0.06666666666666667, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 15, "brier_mean": 1.0722878885683507, "ordinal_mae": 1.7834788096655094, "n_strict_valid": 15, "ece_top_label_10_bins": 0.48521827874136436, "latency": { "n": 15, "p50_ms": 146.2897788733244, "p95_ms": 198.87302499264473 } }, "tool_selection": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.03046341695372773, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.13008134249228204, "latency": { "n": 12, "p50_ms": 166.70902259647846, "p95_ms": 169.03774179518223 } }, "tradeoff": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 2, "accuracy": 0.3333333333333333, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 1.2060304057727296, "ordinal_mae": null, "n_strict_valid": 6, "ece_top_label_10_bins": 0.7290102554730293, "latency": { "n": 6, "p50_ms": 169.56037376075983, "p95_ms": 193.40453296899796 } }, "trap": { "n_planned": 8, "n_attempted": 8, "n_scorable": 8, "n_valid": 8, "n_correct": 7, "accuracy": 0.875, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 8, "brier_mean": 0.13903258666656876, "ordinal_mae": 0.7301031808806691, "n_strict_valid": 8, "ece_top_label_10_bins": 0.15301882710605808, "latency": { "n": 8, "p50_ms": 96.12796269357204, "p95_ms": 160.89465590193865 } } }, "summary_sha256": "3fbd67b643e516713dfab7d7ba54e65aa5ddbb7eb768368f7a0d8c3c4af38750", "model_identity": [ "Qwen/Qwen3.5-2B" ], "price_per_1000_decisions_usd": null, "cost_basis": "local_compute_unmetered_not_free", "request_accounting": { "planned": 231, "attempted": 231, "operational_successes": 231, "failed": 0, "pending": 0, "in_flight": 0 }, "checkpoint_identity": { "model": "Qwen/Qwen3.5-2B", "method": "lora_decision_head", "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "temperature": 1.518796342858676, "code_commit": "3c5bb4a5451ccfd81a5b9f2e05edaf2103805c94", "max_length": 16384 }, "model_url": "https://huggingface.co/ZefanCai/Open-Jev-2B" }, { "id": "9b", "label": "Released Open-Jev 9B", "status": "complete", "audit_status": "passed", "probability_source": "native", "release_status": "previously_released_checkpoint", "metrics": { "n_planned": 231, "n_attempted": 231, "n_scorable": 231, "n_valid": 231, "n_correct": 179, "accuracy": 0.7748917748917749, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 231, "brier_mean": 0.3218882815158104, "ordinal_mae": 0.2349830503995199, "n_strict_valid": 231, "ece_top_label_10_bins": 0.08575426096383554, "latency": { "n": 231, "p50_ms": 189.151581376791, "p95_ms": 839.3336646258831 } }, "per_public_tier": { "original": { "n_planned": 72, "n_attempted": 72, "n_scorable": 72, "n_valid": 72, "n_correct": 65, "accuracy": 0.9027777777777778, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 72, "brier_mean": 0.1687534017432288, "ordinal_mae": 0.045155957036321914, "n_strict_valid": 72, "ece_top_label_10_bins": 0.04388296193006199, "latency": { "n": 72, "p50_ms": 175.4992576315999, "p95_ms": 263.4171257726848 } }, "easy": { "n_planned": 48, "n_attempted": 48, "n_scorable": 48, "n_valid": 48, "n_correct": 48, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 48, "brier_mean": 0.014923372692400932, "ordinal_mae": null, "n_strict_valid": 48, "ece_top_label_10_bins": 0.02494355232850446, "latency": { "n": 48, "p50_ms": 214.85669538378716, "p95_ms": 229.01130113750696 } }, "hard": { "n_planned": 111, "n_attempted": 111, "n_scorable": 111, "n_valid": 111, "n_correct": 66, "accuracy": 0.5945945945945946, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 111, "brier_mean": 0.5539605965351754, "ordinal_mae": 0.6146372371259159, "n_strict_valid": 111, "ece_top_label_10_bins": 0.1979166602721306, "latency": { "n": 111, "p50_ms": 194.74229216575623, "p95_ms": 994.1422892734408 } } }, "per_family": { "adequacy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 10, "accuracy": 0.8333333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.2556265033220004, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.1059673444639138, "latency": { "n": 12, "p50_ms": 52.25101113319397, "p95_ms": 57.29069672524929 } }, "adversarial": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 6, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.017702745952438714, "ordinal_mae": 0.001891970325702319, "n_strict_valid": 6, "ece_top_label_10_bins": 0.04002343553322875, "latency": { "n": 6, "p50_ms": 196.39424979686737, "p95_ms": 203.74302146956325 } }, "ambiguous": { "n_planned": 7, "n_attempted": 7, "n_scorable": 7, "n_valid": 7, "n_correct": 3, "accuracy": 0.42857142857142855, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 7, "brier_mean": 0.6644658006971323, "ordinal_mae": null, "n_strict_valid": 7, "ece_top_label_10_bins": 0.33684231927306985, "latency": { "n": 7, "p50_ms": 148.70204031467438, "p95_ms": 195.35846393555397 } }, "extraction": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 23, "accuracy": 0.9583333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 0.12265361811113107, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.09843498499991149, "latency": { "n": 24, "p50_ms": 175.27149803936481, "p95_ms": 217.33765956014395 } }, "fact": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 1.3636330879491738e-05, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0017469297606330647, "latency": { "n": 12, "p50_ms": 46.17898538708687, "p95_ms": 49.50635628774762 } }, "intent": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 24, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 0.0498163366809978, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.10290271741396406, "latency": { "n": 24, "p50_ms": 218.83696876466274, "p95_ms": 245.76933681964871 } }, "judge_hard": { "n_planned": 17, "n_attempted": 17, "n_scorable": 17, "n_valid": 17, "n_correct": 13, "accuracy": 0.7647058823529411, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 17, "brier_mean": 0.2835286845232439, "ordinal_mae": null, "n_strict_valid": 17, "ece_top_label_10_bins": 0.12352287045988272, "latency": { "n": 17, "p50_ms": 51.84141919016838, "p95_ms": 55.12303188443184 } }, "long_policy": { "n_planned": 19, "n_attempted": 19, "n_scorable": 19, "n_valid": 19, "n_correct": 9, "accuracy": 0.47368421052631576, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 19, "brier_mean": 0.7781885489251826, "ordinal_mae": 0.47884022448122454, "n_strict_valid": 19, "ece_top_label_10_bins": 0.35524147412187135, "latency": { "n": 19, "p50_ms": 721.1358640342951, "p95_ms": 1294.8375727981313 } }, "multi_hop": { "n_planned": 18, "n_attempted": 18, "n_scorable": 18, "n_valid": 18, "n_correct": 12, "accuracy": 0.6666666666666666, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 18, "brier_mean": 0.5368162752202905, "ordinal_mae": 0.6745580713510779, "n_strict_valid": 18, "ece_top_label_10_bins": 0.26304338130637406, "latency": { "n": 18, "p50_ms": 773.4452355653048, "p95_ms": 1003.2202990725636 } }, "ordinal": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.004061181304548735, "ordinal_mae": 0.045155957036321914, "n_strict_valid": 12, "ece_top_label_10_bins": 0.03507090338411857, "latency": { "n": 12, "p50_ms": 175.48021115362644, "p95_ms": 204.46595158427954 } }, "policy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.025087351823678005, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.03684481756583034, "latency": { "n": 12, "p50_ms": 52.30668280273676, "p95_ms": 335.4385999962683 } }, "probability": { "n_planned": 10, "n_attempted": 10, "n_scorable": 10, "n_valid": 10, "n_correct": 6, "accuracy": 0.6, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 10, "brier_mean": 0.6999625753690737, "ordinal_mae": null, "n_strict_valid": 10, "ece_top_label_10_bins": 0.44937269391467005, "latency": { "n": 10, "p50_ms": 163.49549684673548, "p95_ms": 397.06004466861475 } }, "routing": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 8, "accuracy": 0.6666666666666666, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.4419609968596901, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.23727607614753923, "latency": { "n": 12, "p50_ms": 259.2610474675894, "p95_ms": 266.16309117525816 } }, "routing_hard": { "n_planned": 5, "n_attempted": 5, "n_scorable": 5, "n_valid": 5, "n_correct": 5, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 5, "brier_mean": 0.0012142696353332166, "ordinal_mae": null, "n_strict_valid": 5, "ece_top_label_10_bins": 0.024145532255857582, "latency": { "n": 5, "p50_ms": 236.70692183077335, "p95_ms": 248.6088715493679 } }, "temporal_numeric": { "n_planned": 15, "n_attempted": 15, "n_scorable": 15, "n_valid": 15, "n_correct": 2, "accuracy": 0.13333333333333333, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 15, "brier_mean": 0.977167267603474, "ordinal_mae": 1.399478773499916, "n_strict_valid": 15, "ece_top_label_10_bins": 0.3886194503567232, "latency": { "n": 15, "p50_ms": 188.88897076249123, "p95_ms": 387.821365147829 } }, "tool_selection": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.000524322003922427, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.01818651762095147, "latency": { "n": 12, "p50_ms": 217.3369601368904, "p95_ms": 223.44402372837067 } }, "tradeoff": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 2, "accuracy": 0.3333333333333333, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.9312758826812875, "ordinal_mae": null, "n_strict_valid": 6, "ece_top_label_10_bins": 0.46367123940220856, "latency": { "n": 6, "p50_ms": 226.84909589588642, "p95_ms": 275.8756042458117 } }, "trap": { "n_planned": 8, "n_attempted": 8, "n_scorable": 8, "n_valid": 8, "n_correct": 8, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 8, "brier_mean": 0.02662808779547729, "ordinal_mae": 0.6542141586163499, "n_strict_valid": 8, "ece_top_label_10_bins": 0.04189003970381569, "latency": { "n": 8, "p50_ms": 121.89698498696089, "p95_ms": 197.36377084627748 } } }, "summary_sha256": "d6f7e2d50ba0115c2b315868cef218083d88a2dd2555af15f558aa5a75fa4a7b", "model_identity": [ "Qwen/Qwen3.5-9B" ], "price_per_1000_decisions_usd": null, "cost_basis": "local_compute_unmetered_not_free", "request_accounting": { "planned": 231, "attempted": 231, "operational_successes": 231, "failed": 0, "pending": 0, "in_flight": 0 }, "checkpoint_identity": { "model": "Qwen/Qwen3.5-9B", "method": "lora_decision_head", "checkpoint_sha256": "9302c52feba99d079918755f2469f4f088266e9786327bab550248f3d83716d3", "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "temperature": 1.8969118766347646, "code_commit": "3c5bb4a5451ccfd81a5b9f2e05edaf2103805c94", "max_length": 16384 }, "model_url": "https://huggingface.co/ZefanCai/Open-Jev-9B" }, { "id": "jev", "label": "Jev 1.13.0", "status": "complete", "audit_status": "passed", "probability_source": "native", "release_status": "hosted_model", "metrics": { "n_planned": 231, "n_attempted": 231, "n_scorable": 231, "n_valid": 231, "n_correct": 200, "accuracy": 0.8658008658008658, "schema_validity": 1.0, "schema_validity_strict": 0.9956709956709957, "n_renormalized": 1, "operational_success": 1.0, "calibration_n": 231, "brier_mean": 0.181115588699978, "ordinal_mae": 0.14842873176206506, "n_strict_valid": 230, "ece_top_label_10_bins": 0.03175783812147412, "latency": { "n": 231, "p50_ms": 291.3294266909361, "p95_ms": 353.68376038968563 } }, "per_public_tier": { "original": { "n_planned": 72, "n_attempted": 72, "n_scorable": 72, "n_valid": 72, "n_correct": 71, "accuracy": 0.9861111111111112, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 72, "brier_mean": 0.02652777777777777, "ordinal_mae": 0.0016666666666666668, "n_strict_valid": 72, "ece_top_label_10_bins": 0.0358333333333334, "latency": { "n": 72, "p50_ms": 290.4845345765352, "p95_ms": 357.1528069674969 } }, "easy": { "n_planned": 48, "n_attempted": 48, "n_scorable": 48, "n_valid": 48, "n_correct": 48, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 48, "brier_mean": 0.0002166666666666669, "ordinal_mae": null, "n_strict_valid": 48, "ece_top_label_10_bins": 0.004583333333333273, "latency": { "n": 48, "p50_ms": 285.36637127399445, "p95_ms": 333.43319119885564 } }, "hard": { "n_planned": 111, "n_attempted": 111, "n_scorable": 111, "n_valid": 111, "n_correct": 81, "accuracy": 0.7297297297297297, "schema_validity": 1.0, "schema_validity_strict": 0.990990990990991, "n_renormalized": 1, "operational_success": 1.0, "calibration_n": 111, "brier_mean": 0.35961532423148573, "ordinal_mae": 0.4419528619528619, "n_strict_valid": 110, "ece_top_label_10_bins": 0.07491946491946491, "latency": { "n": 111, "p50_ms": 295.5911159515381, "p95_ms": 355.93139473348856 } } }, "per_family": { "adequacy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.019633333333333332, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.07666666666666672, "latency": { "n": 12, "p50_ms": 288.1044028326869, "p95_ms": 332.3359319940209 } }, "adversarial": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 6, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.0024666666666666648, "ordinal_mae": 0.010000000000000009, "n_strict_valid": 6, "ece_top_label_10_bins": 0.020000000000000018, "latency": { "n": 6, "p50_ms": 288.8526879251003, "p95_ms": 318.77397652715445 } }, "ambiguous": { "n_planned": 7, "n_attempted": 7, "n_scorable": 7, "n_valid": 7, "n_correct": 6, "accuracy": 0.8571428571428571, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 7, "brier_mean": 0.2778285714285714, "ordinal_mae": null, "n_strict_valid": 7, "ece_top_label_10_bins": 0.14714285714285716, "latency": { "n": 7, "p50_ms": 273.74886348843575, "p95_ms": 307.69569631665945 } }, "extraction": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 24, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 1.6666666666666684e-05, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.0008333333333333526, "latency": { "n": 24, "p50_ms": 283.81473757326603, "p95_ms": 334.88235268741846 } }, "fact": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0008666666666666676, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0183333333333332, "latency": { "n": 12, "p50_ms": 280.62292002141476, "p95_ms": 328.6488514393568 } }, "intent": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 24, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 0.0001916666666666669, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.004583333333333273, "latency": { "n": 24, "p50_ms": 288.90044894069433, "p95_ms": 336.73174157738686 } }, "judge_hard": { "n_planned": 17, "n_attempted": 17, "n_scorable": 17, "n_valid": 17, "n_correct": 12, "accuracy": 0.7058823529411765, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 17, "brier_mean": 0.3796470588235294, "ordinal_mae": null, "n_strict_valid": 17, "ece_top_label_10_bins": 0.128235294117647, "latency": { "n": 17, "p50_ms": 282.56505727767944, "p95_ms": 338.80397006869316 } }, "long_policy": { "n_planned": 19, "n_attempted": 19, "n_scorable": 19, "n_valid": 19, "n_correct": 12, "accuracy": 0.631578947368421, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 19, "brier_mean": 0.4931684210526316, "ordinal_mae": 0.9249999999999999, "n_strict_valid": 19, "ece_top_label_10_bins": 0.2189473684210526, "latency": { "n": 19, "p50_ms": 334.5989026129246, "p95_ms": 390.30216354876734 } }, "multi_hop": { "n_planned": 18, "n_attempted": 18, "n_scorable": 18, "n_valid": 18, "n_correct": 16, "accuracy": 0.8888888888888888, "schema_validity": 1.0, "schema_validity_strict": 0.9444444444444444, "n_renormalized": 1, "operational_success": 1.0, "calibration_n": 18, "brier_mean": 0.22512783276082937, "ordinal_mae": 0.17171717171717127, "n_strict_valid": 17, "ece_top_label_10_bins": 0.11922558922558926, "latency": { "n": 18, "p50_ms": 292.0953091233969, "p95_ms": 327.91752312332386 } }, "ordinal": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 3.333333333333337e-05, "ordinal_mae": 0.0016666666666666668, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0016666666666665941, "latency": { "n": 12, "p50_ms": 286.93894762545824, "p95_ms": 364.1756023280322 } }, "policy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 11, "accuracy": 0.9166666666666666, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.13903333333333331, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.12333333333333334, "latency": { "n": 12, "p50_ms": 301.6517860814929, "p95_ms": 342.3321493901312 } }, "probability": { "n_planned": 10, "n_attempted": 10, "n_scorable": 10, "n_valid": 10, "n_correct": 7, "accuracy": 0.7, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 10, "brier_mean": 0.3006, "ordinal_mae": null, "n_strict_valid": 10, "ece_top_label_10_bins": 0.168, "latency": { "n": 10, "p50_ms": 308.0543326213956, "p95_ms": 348.7431760877371 } }, "routing": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 5.000000000000005e-05, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0024999999999999467, "latency": { "n": 12, "p50_ms": 276.76732931286097, "p95_ms": 393.7124431133269 } }, "routing_hard": { "n_planned": 5, "n_attempted": 5, "n_scorable": 5, "n_valid": 5, "n_correct": 5, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 5, "brier_mean": 0.0, "ordinal_mae": null, "n_strict_valid": 5, "ece_top_label_10_bins": 0.0, "latency": { "n": 5, "p50_ms": 289.12290930747986, "p95_ms": 323.3253136277199 } }, "temporal_numeric": { "n_planned": 15, "n_attempted": 15, "n_scorable": 15, "n_valid": 15, "n_correct": 4, "accuracy": 0.26666666666666666, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 15, "brier_mean": 0.8547066666666667, "ordinal_mae": 0.3900000000000001, "n_strict_valid": 15, "ece_top_label_10_bins": 0.29133333333333333, "latency": { "n": 15, "p50_ms": 298.8148834556341, "p95_ms": 333.84816385805607 } }, "tool_selection": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0, "latency": { "n": 12, "p50_ms": 288.7122454121709, "p95_ms": 312.70830640569324 } }, "tradeoff": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 5, "accuracy": 0.8333333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.36696666666666666, "ordinal_mae": null, "n_strict_valid": 6, "ece_top_label_10_bins": 0.3083333333333333, "latency": { "n": 6, "p50_ms": 307.3793686926365, "p95_ms": 323.9824348129332 } }, "trap": { "n_planned": 8, "n_attempted": 8, "n_scorable": 8, "n_valid": 8, "n_correct": 8, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 8, "brier_mean": 0.006600000000000001, "ordinal_mae": 0.23, "n_strict_valid": 8, "ece_top_label_10_bins": 0.042500000000000024, "latency": { "n": 8, "p50_ms": 280.4714208468795, "p95_ms": 293.25760770589113 } } }, "summary_sha256": "0b5a6ced50e08a6afd3cb76971b1f095fdc6444a11ae7b663cc020d2b344b71a", "model_identity": [ "jev-1.13.0" ], "price_per_1000_decisions_usd": 0.038419818181818166, "cost_basis": "derived_usage_times_tariff", "request_accounting": { "planned": 231, "attempted": 231, "operational_successes": 231, "failed": 0, "pending": 0, "in_flight": 0 }, "request_options": {} }, { "id": "luna", "label": "GPT-5.6 Luna", "status": "complete", "audit_status": "passed", "probability_source": "verbalized", "release_status": "hosted_model", "metrics": { "n_planned": 231, "n_attempted": 231, "n_scorable": 231, "n_valid": 231, "n_correct": 206, "accuracy": 0.8917748917748918, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 231, "brier_mean": 0.20743182410112307, "ordinal_mae": 0.2897777777777778, "n_strict_valid": 231, "ece_top_label_10_bins": 0.09322807293722962, "latency": { "n": 231, "p50_ms": 953.8466669619083, "p95_ms": 1307.5056709349155 } }, "per_public_tier": { "original": { "n_planned": 72, "n_attempted": 72, "n_scorable": 72, "n_valid": 72, "n_correct": 69, "accuracy": 0.9583333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 72, "brier_mean": 0.08216637805555554, "ordinal_mae": 0.00833333333333334, "n_strict_valid": 72, "ece_top_label_10_bins": 0.0325347222222222, "latency": { "n": 72, "p50_ms": 905.5536277592182, "p95_ms": 1303.876217827201 } }, "easy": { "n_planned": 48, "n_attempted": 48, "n_scorable": 48, "n_valid": 48, "n_correct": 48, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 48, "brier_mean": 9.375000000000015e-05, "ordinal_mae": null, "n_strict_valid": 48, "ece_top_label_10_bins": 0.0029999999999997806, "latency": { "n": 48, "p50_ms": 1029.3054897338152, "p95_ms": 1315.8735093660653 } }, "hard": { "n_planned": 111, "n_attempted": 111, "n_scorable": 111, "n_valid": 111, "n_correct": 89, "accuracy": 0.8018018018018018, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 111, "brier_mean": 0.37834479412035527, "ordinal_mae": 0.8526666666666668, "n_strict_valid": 111, "ece_top_label_10_bins": 0.17420887250900846, "latency": { "n": 111, "p50_ms": 957.7288702130318, "p95_ms": 1297.3956298083067 } } }, "per_family": { "adequacy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 10, "accuracy": 0.8333333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.3234668333333333, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.1615833333333333, "latency": { "n": 12, "p50_ms": 597.6372500881553, "p95_ms": 949.9158470891416 } }, "adversarial": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 6, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.007567000000000003, "ordinal_mae": 0.15000000000000013, "n_strict_valid": 6, "ece_top_label_10_bins": 0.026833333333333226, "latency": { "n": 6, "p50_ms": 924.7055305168033, "p95_ms": 1125.237537547946 } }, "ambiguous": { "n_planned": 7, "n_attempted": 7, "n_scorable": 7, "n_valid": 7, "n_correct": 6, "accuracy": 0.8571428571428571, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 7, "brier_mean": 0.2817571428571429, "ordinal_mae": null, "n_strict_valid": 7, "ece_top_label_10_bins": 0.1328571428571429, "latency": { "n": 7, "p50_ms": 944.8023904114962, "p95_ms": 1069.3884667009115 } }, "extraction": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 24, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 0.0, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.0, "latency": { "n": 24, "p50_ms": 887.511532753706, "p95_ms": 1230.9859859757125 } }, "fact": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0, "latency": { "n": 12, "p50_ms": 605.4248455911875, "p95_ms": 712.7517947927116 } }, "intent": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 24, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 0.0006703750000000008, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.015916666666666912, "latency": { "n": 24, "p50_ms": 985.5544334277511, "p95_ms": 2206.623049546031 } }, "judge_hard": { "n_planned": 17, "n_attempted": 17, "n_scorable": 17, "n_valid": 17, "n_correct": 16, "accuracy": 0.9411764705882353, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 17, "brier_mean": 0.117518, "ordinal_mae": null, "n_strict_valid": 17, "ece_top_label_10_bins": 0.0533529411764706, "latency": { "n": 17, "p50_ms": 669.2503429949284, "p95_ms": 835.5219118297098 } }, "long_policy": { "n_planned": 19, "n_attempted": 19, "n_scorable": 19, "n_valid": 19, "n_correct": 16, "accuracy": 0.8421052631578947, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 19, "brier_mean": 0.30945143052631585, "ordinal_mae": 0.975, "n_strict_valid": 19, "ece_top_label_10_bins": 0.13968421052631597, "latency": { "n": 19, "p50_ms": 1033.8401086628437, "p95_ms": 1258.3951881155372 } }, "multi_hop": { "n_planned": 18, "n_attempted": 18, "n_scorable": 18, "n_valid": 18, "n_correct": 15, "accuracy": 0.8333333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 18, "brier_mean": 0.33485563888888886, "ordinal_mae": 1.0, "n_strict_valid": 18, "ece_top_label_10_bins": 0.14655555555555566, "latency": { "n": 18, "p50_ms": 1135.3037618100643, "p95_ms": 1428.6073806695626 } }, "ordinal": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0009666666666666678, "ordinal_mae": 0.00833333333333334, "n_strict_valid": 12, "ece_top_label_10_bins": 0.015833333333333477, "latency": { "n": 12, "p50_ms": 964.6959016099572, "p95_ms": 1219.1477959975598 } }, "policy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 11, "accuracy": 0.9166666666666666, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.16751666666666667, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.07416666666666671, "latency": { "n": 12, "p50_ms": 833.9387457817793, "p95_ms": 1607.7242239378388 } }, "probability": { "n_planned": 10, "n_attempted": 10, "n_scorable": 10, "n_valid": 10, "n_correct": 7, "accuracy": 0.7, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 10, "brier_mean": 0.4683533967359449, "ordinal_mae": null, "n_strict_valid": 10, "ece_top_label_10_bins": 0.17961848485, "latency": { "n": 10, "p50_ms": 902.0320000126958, "p95_ms": 1139.0445513650773 } }, "routing": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 7.37683333333334e-05, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0030416666666667203, "latency": { "n": 12, "p50_ms": 1193.863115273416, "p95_ms": 1313.8467607088387 } }, "routing_hard": { "n_planned": 5, "n_attempted": 5, "n_scorable": 5, "n_valid": 5, "n_correct": 5, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 5, "brier_mean": 0.0008740000000000012, "ordinal_mae": null, "n_strict_valid": 5, "ece_top_label_10_bins": 0.02300000000000002, "latency": { "n": 5, "p50_ms": 1224.1442762315273, "p95_ms": 1378.5467159003017 } }, "temporal_numeric": { "n_planned": 15, "n_attempted": 15, "n_scorable": 15, "n_valid": 15, "n_correct": 5, "accuracy": 0.3333333333333333, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 15, "brier_mean": 1.3290454666666667, "ordinal_mae": 2.0, "n_strict_valid": 15, "ece_top_label_10_bins": 0.6626000000000001, "latency": { "n": 15, "p50_ms": 1068.2277288287878, "p95_ms": 1227.3321516811845 } }, "tool_selection": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 8.583333333333345e-06, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0018333333333332424, "latency": { "n": 12, "p50_ms": 1134.887303225696, "p95_ms": 1311.3228523172438 } }, "tradeoff": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 5, "accuracy": 0.8333333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.24163524999999997, "ordinal_mae": null, "n_strict_valid": 6, "ece_top_label_10_bins": 0.15383333333333338, "latency": { "n": 6, "p50_ms": 1109.9263867363334, "p95_ms": 1373.6321236938238 } }, "trap": { "n_planned": 8, "n_attempted": 8, "n_scorable": 8, "n_valid": 8, "n_correct": 8, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 8, "brier_mean": 4.8500000000000054e-05, "ordinal_mae": 0.016, "n_strict_valid": 8, "ece_top_label_10_bins": 0.0034999999999999476, "latency": { "n": 8, "p50_ms": 847.6508380845189, "p95_ms": 1195.921442937106 } } }, "summary_sha256": "2923d9f6a6de28939c12bedb79fb0caba8e5ab7c7636aa2875c1d4ac47c6c40a", "model_identity": [ "gpt-5.6-luna" ], "price_per_1000_decisions_usd": 0.179135064935065, "cost_basis": "derived_usage_times_tariff", "request_accounting": { "planned": 231, "attempted": 231, "operational_successes": 231, "failed": 0, "pending": 0, "in_flight": 0 }, "request_options": { "max_completion_tokens": 4096, "reasoning_effort": "none" } }, { "id": "astra", "label": "GPT-6 Astra", "status": "complete", "audit_status": "passed", "probability_source": "verbalized", "release_status": "hosted_model", "metrics": { "n_planned": 231, "n_attempted": 231, "n_scorable": 231, "n_valid": 231, "n_correct": 231, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 231, "brier_mean": 0.008457748053556388, "ordinal_mae": 0.0002666666666666692, "n_strict_valid": 231, "ece_top_label_10_bins": 0.01494360581698253, "latency": { "n": 231, "p50_ms": 2206.3674051314592, "p95_ms": 3581.649805419147 } }, "per_public_tier": { "original": { "n_planned": 72, "n_attempted": 72, "n_scorable": 72, "n_valid": 72, "n_correct": 72, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 72, "brier_mean": 9.122777777777793e-06, "ordinal_mae": 0.0, "n_strict_valid": 72, "ece_top_label_10_bins": 0.0009374999999999245, "latency": { "n": 72, "p50_ms": 2073.2710557058454, "p95_ms": 3334.7264141775668 } }, "easy": { "n_planned": 48, "n_attempted": 48, "n_scorable": 48, "n_valid": 48, "n_correct": 48, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 48, "brier_mean": 4.354166666666695e-07, "ordinal_mae": null, "n_strict_valid": 48, "ece_top_label_10_bins": 0.00014583333333328952, "latency": { "n": 48, "p50_ms": 2065.2905497699976, "p95_ms": 2597.940638381988 } }, "hard": { "n_planned": 111, "n_attempted": 111, "n_scorable": 111, "n_valid": 111, "n_correct": 111, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 111, "brier_mean": 0.01759515369704077, "ordinal_mae": 0.0008000000000000076, "n_strict_valid": 111, "ece_top_label_10_bins": 0.030427684177684277, "latency": { "n": 111, "p50_ms": 2409.812716767192, "p95_ms": 3764.902842231095 } } }, "per_family": { "adequacy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 1.733333333333335e-05, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0011666666666666492, "latency": { "n": 12, "p50_ms": 1890.8640947192907, "p95_ms": 2763.2116903550914 } }, "adversarial": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 6, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 0.0, "ordinal_mae": 0.0, "n_strict_valid": 6, "ece_top_label_10_bins": 0.0, "latency": { "n": 6, "p50_ms": 2045.9959134459496, "p95_ms": 2424.451806116849 } }, "ambiguous": { "n_planned": 7, "n_attempted": 7, "n_scorable": 7, "n_valid": 7, "n_correct": 7, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 7, "brier_mean": 1.7142857142857162e-06, "ordinal_mae": null, "n_strict_valid": 7, "ece_top_label_10_bins": 0.0005714285714285561, "latency": { "n": 7, "p50_ms": 2214.300887659192, "p95_ms": 2326.7313733696938 } }, "extraction": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 24, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 5.000000000000006e-07, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.00012500000000004174, "latency": { "n": 24, "p50_ms": 2270.363776013255, "p95_ms": 3301.2143832631405 } }, "fact": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0, "latency": { "n": 12, "p50_ms": 1741.9250048696995, "p95_ms": 2051.1504320427775 } }, "intent": { "n_planned": 24, "n_attempted": 24, "n_scorable": 24, "n_valid": 24, "n_correct": 24, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 24, "brier_mean": 6.260000000000017e-06, "ordinal_mae": null, "n_strict_valid": 24, "ece_top_label_10_bins": 0.0013125000000000497, "latency": { "n": 24, "p50_ms": 2288.251710124314, "p95_ms": 3441.962145175784 } }, "judge_hard": { "n_planned": 17, "n_attempted": 17, "n_scorable": 17, "n_valid": 17, "n_correct": 17, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 17, "brier_mean": 9.411764705882361e-07, "ordinal_mae": null, "n_strict_valid": 17, "ece_top_label_10_bins": 0.00035294117647044487, "latency": { "n": 17, "p50_ms": 2164.8322381079197, "p95_ms": 3375.6031457334757 } }, "long_policy": { "n_planned": 19, "n_attempted": 19, "n_scorable": 19, "n_valid": 19, "n_correct": 19, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 19, "brier_mean": 1.3852631578947387e-06, "ordinal_mae": 0.0008000000000000229, "n_strict_valid": 19, "ece_top_label_10_bins": 0.00047368421052629284, "latency": { "n": 19, "p50_ms": 2697.408674284816, "p95_ms": 3920.9136612713332 } }, "multi_hop": { "n_planned": 18, "n_attempted": 18, "n_scorable": 18, "n_valid": 18, "n_correct": 18, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 18, "brier_mean": 1.8888888888888916e-06, "ordinal_mae": 0.0, "n_strict_valid": 18, "ece_top_label_10_bins": 0.00038888888888877204, "latency": { "n": 18, "p50_ms": 2366.9741228222847, "p95_ms": 2879.5858703553677 } }, "ordinal": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0, "ordinal_mae": 0.0, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0, "latency": { "n": 12, "p50_ms": 2024.6059130877256, "p95_ms": 3024.1289761848743 } }, "policy": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 1.6666666666666684e-05, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0008333333333333526, "latency": { "n": 12, "p50_ms": 1864.8706264793873, "p95_ms": 2213.8772747479375 } }, "probability": { "n_planned": 10, "n_attempted": 10, "n_scorable": 10, "n_valid": 10, "n_correct": 10, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 10, "brier_mean": 0.19528628003715257, "ordinal_mae": null, "n_strict_valid": 10, "ece_top_label_10_bins": 0.3327472943722944, "latency": { "n": 10, "p50_ms": 2752.2059362381697, "p95_ms": 3598.1958916410804 } }, "routing": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 8.958333333333346e-06, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0013333333333332975, "latency": { "n": 12, "p50_ms": 2310.5893498286605, "p95_ms": 3567.3369877971704 } }, "routing_hard": { "n_planned": 5, "n_attempted": 5, "n_scorable": 5, "n_valid": 5, "n_correct": 5, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 5, "brier_mean": 1.9452000000000023e-05, "ordinal_mae": null, "n_strict_valid": 5, "ece_top_label_10_bins": 0.0036000000000001586, "latency": { "n": 5, "p50_ms": 2250.106180086732, "p95_ms": 3213.795629143715 } }, "temporal_numeric": { "n_planned": 15, "n_attempted": 15, "n_scorable": 15, "n_valid": 15, "n_correct": 15, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 15, "brier_mean": 2.6666666666666694e-07, "ordinal_mae": 0.0, "n_strict_valid": 15, "ece_top_label_10_bins": 0.00013333333333320763, "latency": { "n": 15, "p50_ms": 2882.8944955021143, "p95_ms": 4454.183423146604 } }, "tool_selection": { "n_planned": 12, "n_attempted": 12, "n_scorable": 12, "n_valid": 12, "n_correct": 12, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 12, "brier_mean": 0.0, "ordinal_mae": null, "n_strict_valid": 12, "ece_top_label_10_bins": 0.0, "latency": { "n": 12, "p50_ms": 2117.344400845468, "p95_ms": 2494.0581970848143 } }, "tradeoff": { "n_planned": 6, "n_attempted": 6, "n_scorable": 6, "n_valid": 6, "n_correct": 6, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 6, "brier_mean": 6.666666666666673e-07, "ordinal_mae": null, "n_strict_valid": 6, "ece_top_label_10_bins": 0.0003333333333332966, "latency": { "n": 6, "p50_ms": 2185.907385312021, "p95_ms": 2468.9954281784594 } }, "trap": { "n_planned": 8, "n_attempted": 8, "n_scorable": 8, "n_valid": 8, "n_correct": 8, "accuracy": 1.0, "schema_validity": 1.0, "schema_validity_strict": 1.0, "n_renormalized": 0, "operational_success": 1.0, "calibration_n": 8, "brier_mean": 7.100000000000008e-07, "ordinal_mae": 0.0031999999999999997, "n_strict_valid": 8, "ece_top_label_10_bins": 0.00024999999999997247, "latency": { "n": 8, "p50_ms": 2219.6049727499485, "p95_ms": 2547.8594036772843 } } }, "summary_sha256": "8d7c5989263195311115822c21aa252178ceb1281273fea0b57fb14bc741a29b", "model_identity": [ "gpt-6-astra" ], "price_per_1000_decisions_usd": 8.81978354978355, "cost_basis": "derived_usage_times_tariff", "request_accounting": { "planned": 231, "attempted": 231, "operational_successes": 231, "failed": 0, "pending": 0, "in_flight": 0 }, "request_options": { "max_completion_tokens": 4096, "reasoning_effort": "low" } } ], "audit": { "openjev": { "status": "passed", "artifact": "openjev-independent-replay-audit.json", "sha256": "e89aeb89aae29f3f9f804d22488a3716a8b22b8d253eea552a79f89545adf111" }, "hosted_summary_replay": { "status": "passed", "artifact": "hosted-independent-summary-audit.json", "sha256": "a3a04d1f3bd2a2c26013de28bd027523c9593c98b80ddd3fce38410672ef6d2c" }, "hosted_raw_replay": { "status": "passed", "artifact": "hosted-raw-replay-audit.json", "sha256": "0bd587ad999154a3d91dcca2311218462d1d0822bbbfca4d11923d3f5f7d4daa" } }, "next_stage": { "status": "being_prepared", "description": "A new 27B training iteration with more diverse training data is being prepared.", "new_27b_training_resumed": false, "new_27b_training_completed": false, "note": "The results above use the already released 2B/9B baselines and hosted models." }, "method_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/docs/jevbench-public.md", "report_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/jevbench-public-20260921/baseline-summary.json", "notes": [ "This evaluates the already released 2B and 9B adapters and decision heads; these are not newly trained checkpoints.", "The 231-task public subset is separate from the existing 76-case provider comparison and release-v2 test/OOD evaluation.", "No raw task text, states, labels, task IDs, provider responses, credentials, or absolute node paths are exported." ] }