{ "schema_version": 1, "generated_at": "2026-09-21T01:13:58.305852+00:00", "status": "partial", "providers": [ { "id": "2b", "name": "Open-Jev-2B" }, { "id": "9b", "name": "Open-Jev-9B" }, { "id": "jev", "name": "Jev 1.13.0" }, { "id": "luna", "name": "GPT-5.6 Luna (none)" }, { "id": "astra", "name": "GPT-6 Astra (low)" } ], "suites": [ { "id": "matched-coverage", "label": "Same 76 coverage decisions", "hard_targets": 76, "soft_targets": 0, "requests": 76, "note": "Exactly the same hard cases for every provider. Released 2B/9B predictions are reused after exact row/checkpoint/hash verification; they are not new inference or latency measurements.", "results": { "2b": { "status": "complete", "correct": 65, "total": 76, "planned": 76, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/2b-coverage-quality-archive.json" }, "9b": { "status": "complete", "correct": 72, "total": 76, "planned": 76, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/9b-coverage-quality-archive.json" }, "jev": { "status": "complete", "correct": 66, "total": 76, "planned": 76, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 1, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/jev-coverage-quality.json" }, "luna": { "status": "complete", "correct": 60, "total": 76, "planned": 76, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-5.6-luna-coverage-quality.json" }, "astra": { "status": "complete", "correct": 71, "total": 76, "planned": 76, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-6-astra-coverage-quality.json" } } }, { "id": "coverage", "label": "Broader domain coverage", "hard_targets": 140, "soft_targets": 6, "requests": 189, "note": "146 labelled decisions and 43 unlabelled saved examples. Six soft targets and the unlabelled examples are excluded from hard accuracy. The released models still need 64 newer hard cases.", "results": { "2b": { "status": "partial", "correct": 65, "total": 76, "planned": 140, "pending": 64, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/2b-coverage-quality-archive.json", "by_source": { "customer-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 11, "hard_correct": 11, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.9583333333333334 }, "painting-geometry-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "hard_correct": 10, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "reasoning-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "hard_correct": 10, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "snake-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 7, "hard_correct": 5, "hard_accuracy_including_errors": 0.7142857142857143, "soft_targets": 1, "expected_accuracy_including_errors": 0.6666666666666666 }, "tic_tac_toe-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 1, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.6666666666666666 }, "tile_platformer-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "hard_correct": 2, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "trex_runner-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "hard_correct": 0, "hard_accuracy_including_errors": 0.0, "soft_targets": 0, "expected_accuracy_including_errors": 0.0 }, "vizdoom-basic-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "hard_correct": 11, "hard_accuracy_including_errors": 0.9166666666666666, "soft_targets": 0, "expected_accuracy_including_errors": 0.9166666666666666 }, "wikispeedia-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 1, "hard_correct": 0, "hard_accuracy_including_errors": 0.0, "soft_targets": 3, "expected_accuracy_including_errors": 0.11458333333333333 }, "workflow-controls-v1/agent_trace_observability": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/customer_service": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "workflow-controls-v1/invoice_processing": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/security_incidents": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } }, "9b": { "status": "partial", "correct": 72, "total": 76, "planned": 140, "pending": 64, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/9b-coverage-quality-archive.json", "by_source": { "customer-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 11, "hard_correct": 11, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.9583333333333334 }, "painting-geometry-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "hard_correct": 11, "hard_accuracy_including_errors": 0.9166666666666666, "soft_targets": 0, "expected_accuracy_including_errors": 0.9166666666666666 }, "reasoning-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "hard_correct": 12, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "snake-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 7, "hard_correct": 7, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.9166666666666666 }, "tic_tac_toe-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 1, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.6666666666666666 }, "tile_platformer-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "hard_correct": 2, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "trex_runner-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "hard_correct": 1, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "vizdoom-basic-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "hard_correct": 11, "hard_accuracy_including_errors": 0.9166666666666666, "soft_targets": 0, "expected_accuracy_including_errors": 0.9166666666666666 }, "wikispeedia-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 1, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 3, "expected_accuracy_including_errors": 0.3645833333333333 }, "workflow-controls-v1/agent_trace_observability": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/customer_service": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "workflow-controls-v1/invoice_processing": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/security_incidents": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } }, "jev": { "status": "complete", "correct": 117, "total": 140, "planned": 140, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 2, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/jev-coverage-quality.json", "by_source": { "amount-extraction-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "browser-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 2, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "citation-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "context-retention-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "customer-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 11, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.9090909090909091, "soft_targets": 1, "expected_accuracy_including_errors": 0.875 }, "drone-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 1, "hard_correct": 6, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "email-selection-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "entity-alignment-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "painting-geometry-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "phone-extraction-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "reasoning-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 11, "hard_accuracy_including_errors": 0.9166666666666666, "soft_targets": 0, "expected_accuracy_including_errors": 0.9166666666666666 }, "silent-failure-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "snake-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 7, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 7, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.9166666666666666 }, "sponsor-segment-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "tic_tac_toe-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 1, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.6666666666666666 }, "tile_platformer-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "usable_decisions_with_probability_mass_failure": 1, "hard_correct": 0, "hard_accuracy_including_errors": 0.0, "soft_targets": 0, "expected_accuracy_including_errors": 0.0 }, "trex_runner-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 1, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "vizdoom-basic-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "wikispeedia-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 1, "usable_decisions_with_probability_mass_failure": 1, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 3, "expected_accuracy_including_errors": 0.3645833333333333 }, "workflow-controls-v1/agent_trace_observability": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/customer_service": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "workflow-controls-v1/invoice_processing": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/security_incidents": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } }, "luna": { "status": "complete", "correct": 109, "total": 140, "planned": 140, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-5.6-luna-coverage-quality.json", "by_source": { "amount-extraction-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 7, "hard_accuracy_including_errors": 0.875, "soft_targets": 0, "expected_accuracy_including_errors": 0.875 }, "browser-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 2, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "citation-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "context-retention-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "customer-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 11, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.9090909090909091, "soft_targets": 1, "expected_accuracy_including_errors": 0.875 }, "drone-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 5, "hard_accuracy_including_errors": 0.4166666666666667, "soft_targets": 0, "expected_accuracy_including_errors": 0.4166666666666667 }, "email-selection-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "entity-alignment-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "painting-geometry-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 9, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "phone-extraction-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "reasoning-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "silent-failure-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "snake-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 7, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 6, "hard_accuracy_including_errors": 0.8571428571428571, "soft_targets": 1, "expected_accuracy_including_errors": 0.7916666666666666 }, "sponsor-segment-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "tic_tac_toe-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 1, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 0, "hard_accuracy_including_errors": 0.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.16666666666666666 }, "tile_platformer-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 1, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "trex_runner-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 2, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "vizdoom-basic-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 9, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "wikispeedia-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 1, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 3, "expected_accuracy_including_errors": 0.28125 }, "workflow-controls-v1/agent_trace_observability": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "workflow-controls-v1/customer_service": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "workflow-controls-v1/invoice_processing": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 }, "workflow-controls-v1/security_incidents": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 3, "hard_accuracy_including_errors": 0.75, "soft_targets": 0, "expected_accuracy_including_errors": 0.75 } } }, "astra": { "status": "complete", "correct": 135, "total": 140, "planned": 140, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-6-astra-coverage-quality.json", "by_source": { "amount-extraction-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "browser-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "citation-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "context-retention-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "customer-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 11, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.9090909090909091, "soft_targets": 1, "expected_accuracy_including_errors": 0.875 }, "drone-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 12, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "email-selection-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "entity-alignment-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "painting-geometry-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 12, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "phone-extraction-control-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 8, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "reasoning-control-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 12, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "silent-failure-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "snake-v1": { "evaluated": 8, "successful_responses": 8, "errors": 0, "hard_targets": 7, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 7, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.9166666666666666 }, "sponsor-segment-control-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "tic_tac_toe-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 1, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 1, "expected_accuracy_including_errors": 0.6666666666666666 }, "tile_platformer-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 0, "hard_accuracy_including_errors": 0.0, "soft_targets": 0, "expected_accuracy_including_errors": 0.0 }, "trex_runner-v1": { "evaluated": 2, "successful_responses": 2, "errors": 0, "hard_targets": 2, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 2, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "vizdoom-basic-v1": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 12, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "wikispeedia-v1": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 1, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 1, "hard_accuracy_including_errors": 1.0, "soft_targets": 3, "expected_accuracy_including_errors": 0.3645833333333333 }, "workflow-controls-v1/agent_trace_observability": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/customer_service": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/invoice_processing": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "workflow-controls-v1/security_incidents": { "evaluated": 4, "successful_responses": 4, "errors": 0, "hard_targets": 4, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 4, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } } } }, { "id": "jf100", "label": "Jev Frontier 100", "hard_targets": 300, "soft_targets": 0, "requests": 300, "note": "100 external holdout items in three candidate rotations; 300 correlated decisions, not 300 independent problems.", "results": { "2b": { "status": "pending", "correct": 0, "total": 0, "planned": 300, "pending": 300, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "9b": { "status": "pending", "correct": 0, "total": 0, "planned": 300, "pending": 300, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "jev": { "status": "complete", "correct": 232, "total": 300, "planned": 300, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 2, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/jev-jf100-quality.json", "by_source": { "jf100/algorithms": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 15, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 }, "jf100/code_semantics": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/customer_service": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 27, "hard_accuracy_including_errors": 0.9, "soft_targets": 0, "expected_accuracy_including_errors": 0.9 }, "jf100/discourse": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/evidence_integration": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 1, "hard_correct": 20, "hard_accuracy_including_errors": 0.6666666666666666, "soft_targets": 0, "expected_accuracy_including_errors": 0.6666666666666666 }, "jf100/formal_logic": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/mathematics": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 25, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "jf100/policy_rules": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/relations": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 10, "hard_accuracy_including_errors": 0.3333333333333333, "soft_targets": 0, "expected_accuracy_including_errors": 0.3333333333333333 }, "jf100/temporal": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 1, "hard_correct": 15, "hard_accuracy_including_errors": 0.5, "soft_targets": 0, "expected_accuracy_including_errors": 0.5 } } }, "luna": { "status": "complete", "correct": 227, "total": 300, "planned": 300, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-5.6-luna-jf100-quality.json", "by_source": { "jf100/algorithms": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 17, "hard_accuracy_including_errors": 0.5666666666666667, "soft_targets": 0, "expected_accuracy_including_errors": 0.5666666666666667 }, "jf100/code_semantics": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/customer_service": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 29, "hard_accuracy_including_errors": 0.9666666666666667, "soft_targets": 0, "expected_accuracy_including_errors": 0.9666666666666667 }, "jf100/discourse": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/evidence_integration": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 25, "hard_accuracy_including_errors": 0.8333333333333334, "soft_targets": 0, "expected_accuracy_including_errors": 0.8333333333333334 }, "jf100/formal_logic": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 28, "hard_accuracy_including_errors": 0.9333333333333333, "soft_targets": 0, "expected_accuracy_including_errors": 0.9333333333333333 }, "jf100/mathematics": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 9, "hard_accuracy_including_errors": 0.3, "soft_targets": 0, "expected_accuracy_including_errors": 0.3 }, "jf100/policy_rules": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/relations": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 12, "hard_accuracy_including_errors": 0.4, "soft_targets": 0, "expected_accuracy_including_errors": 0.4 }, "jf100/temporal": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 17, "hard_accuracy_including_errors": 0.5666666666666667, "soft_targets": 0, "expected_accuracy_including_errors": 0.5666666666666667 } } }, "astra": { "status": "complete", "correct": 300, "total": 300, "planned": 300, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-6-astra-jf100-quality.json", "by_source": { "jf100/algorithms": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/code_semantics": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/customer_service": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/discourse": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/evidence_integration": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/formal_logic": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/mathematics": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/policy_rules": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/relations": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "jf100/temporal": { "evaluated": 30, "successful_responses": 30, "errors": 0, "hard_targets": 30, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 30, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } } } }, { "id": "ir-pilot", "label": "Graded retrieval controls", "hard_targets": 165, "soft_targets": 9, "requests": 132, "note": "Six original queries across three families, with 8 passages per query. Hard categorical decisions only; nine tied-best soft targets are separate. This is not TREC or full heap-reranking quality.", "results": { "2b": { "status": "pending", "correct": 0, "total": 0, "planned": 165, "pending": 165, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "9b": { "status": "pending", "correct": 0, "total": 0, "planned": 165, "pending": 165, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "jev": { "status": "complete", "correct": 165, "total": 165, "planned": 165, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 1, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/jev-ir-pilot-quality.json", "by_source": { "ir/listwise_choice": { "evaluated": 6, "successful_responses": 6, "errors": 0, "hard_targets": 6, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 6, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/listwise_score": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 48, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/pairwise": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 6, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 6, "hard_accuracy_including_errors": 1.0, "soft_targets": 6, "expected_accuracy_including_errors": 0.75 }, "ir/pointwise_noul": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 48, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/pointwise_score": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 1, "hard_correct": 48, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/setwise": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 9, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 9, "hard_accuracy_including_errors": 1.0, "soft_targets": 3, "expected_accuracy_including_errors": 0.8333333333333334 } } }, "luna": { "status": "complete", "correct": 160, "total": 165, "planned": 165, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-5.6-luna-ir-pilot-quality.json", "by_source": { "ir/listwise_choice": { "evaluated": 6, "successful_responses": 6, "errors": 0, "hard_targets": 6, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 6, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/listwise_score": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 47, "hard_accuracy_including_errors": 0.9791666666666666, "soft_targets": 0, "expected_accuracy_including_errors": 0.9791666666666666 }, "ir/pairwise": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 6, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 6, "hard_accuracy_including_errors": 1.0, "soft_targets": 6, "expected_accuracy_including_errors": 0.75 }, "ir/pointwise_noul": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 48, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/pointwise_score": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 45, "hard_accuracy_including_errors": 0.9375, "soft_targets": 0, "expected_accuracy_including_errors": 0.9375 }, "ir/setwise": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 9, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 8, "hard_accuracy_including_errors": 0.8888888888888888, "soft_targets": 3, "expected_accuracy_including_errors": 0.75 } } }, "astra": { "status": "complete", "correct": 165, "total": 165, "planned": 165, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-6-astra-ir-pilot-quality.json", "by_source": { "ir/listwise_choice": { "evaluated": 6, "successful_responses": 6, "errors": 0, "hard_targets": 6, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 6, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/listwise_score": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 48, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/pairwise": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 6, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 6, "hard_accuracy_including_errors": 1.0, "soft_targets": 6, "expected_accuracy_including_errors": 0.75 }, "ir/pointwise_noul": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 48, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/pointwise_score": { "evaluated": 48, "successful_responses": 48, "errors": 0, "hard_targets": 48, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 48, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "ir/setwise": { "evaluated": 12, "successful_responses": 12, "errors": 0, "hard_targets": 9, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 9, "hard_accuracy_including_errors": 1.0, "soft_targets": 3, "expected_accuracy_including_errors": 0.8333333333333334 } } } } }, { "id": "fizzbuzz", "label": "FizzBuzz controls", "hard_targets": 300, "soft_targets": 0, "requests": 100, "note": "Integers 1-100, each with two Nouls and one Choice. Original probe; the community post did not supply a complete prompt to reproduce.", "results": { "2b": { "status": "pending", "correct": 0, "total": 0, "planned": 300, "pending": 300, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "9b": { "status": "pending", "correct": 0, "total": 0, "planned": 300, "pending": 300, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "jev": { "status": "complete", "correct": 299, "total": 300, "planned": 300, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/jev-fizzbuzz-quality.json", "by_source": { "arithmetic-fizzbuzz-control-v1": { "evaluated": 300, "successful_responses": 300, "errors": 0, "hard_targets": 300, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 299, "hard_accuracy_including_errors": 0.9966666666666667, "soft_targets": 0, "expected_accuracy_including_errors": 0.9966666666666667 } } }, "luna": { "status": "complete", "correct": 300, "total": 300, "planned": 300, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-5.6-luna-fizzbuzz-quality.json", "by_source": { "arithmetic-fizzbuzz-control-v1": { "evaluated": 300, "successful_responses": 300, "errors": 0, "hard_targets": 300, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 300, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } }, "astra": { "status": "complete", "correct": 300, "total": 300, "planned": 300, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-6-astra-fizzbuzz-quality.json", "by_source": { "arithmetic-fizzbuzz-control-v1": { "evaluated": 300, "successful_responses": 300, "errors": 0, "hard_targets": 300, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 300, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } } } }, { "id": "mailroom", "label": "Multilingual mailroom controls", "hard_targets": 921, "soft_targets": 0, "requests": 87, "note": "87 requests with 11 heads each. The category of 36 non-bill messages has no gold and is excluded. 921 supervised decisions, with shared families and repeated inputs; not end-to-end email automation.", "results": { "2b": { "status": "pending", "correct": 0, "total": 0, "planned": 921, "pending": 921, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "9b": { "status": "pending", "correct": 0, "total": 0, "planned": 921, "pending": 921, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0 }, "jev": { "status": "complete", "correct": 908, "total": 921, "planned": 921, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/jev-mailroom-quality.json", "by_source": { "mailroom/about_airline": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_banking": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 86, "hard_accuracy_including_errors": 0.9885057471264368, "soft_targets": 0, "expected_accuracy_including_errors": 0.9885057471264368 }, "mailroom/about_education": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_electricity": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_telecom": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/category": { "evaluated": 51, "successful_responses": 51, "errors": 0, "hard_targets": 51, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 49, "hard_accuracy_including_errors": 0.9607843137254902, "soft_targets": 0, "expected_accuracy_including_errors": 0.9607843137254902 }, "mailroom/from_billing_entity": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 78, "hard_accuracy_including_errors": 0.896551724137931, "soft_targets": 0, "expected_accuracy_including_errors": 0.896551724137931 }, "mailroom/has_billing_identifiers": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 86, "hard_accuracy_including_errors": 0.9885057471264368, "soft_targets": 0, "expected_accuracy_including_errors": 0.9885057471264368 }, "mailroom/is_promotional": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/kind": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/states_amount_owed": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } }, "luna": { "status": "complete", "correct": 900, "total": 921, "planned": 921, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-5.6-luna-mailroom-quality.json", "by_source": { "mailroom/about_airline": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_banking": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_education": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_electricity": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_telecom": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/category": { "evaluated": 51, "successful_responses": 51, "errors": 0, "hard_targets": 51, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 50, "hard_accuracy_including_errors": 0.9803921568627451, "soft_targets": 0, "expected_accuracy_including_errors": 0.9803921568627451 }, "mailroom/from_billing_entity": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 73, "hard_accuracy_including_errors": 0.8390804597701149, "soft_targets": 0, "expected_accuracy_including_errors": 0.8390804597701149 }, "mailroom/has_billing_identifiers": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 81, "hard_accuracy_including_errors": 0.9310344827586207, "soft_targets": 0, "expected_accuracy_including_errors": 0.9310344827586207 }, "mailroom/is_promotional": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/kind": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/states_amount_owed": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } }, "astra": { "status": "complete", "correct": 913, "total": 921, "planned": 921, "pending": 0, "errors": 0, "strict_probability_mass_failures_counted_categorically": 0, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/provider-comparison-20260920/openai-gpt-6-astra-mailroom-quality.json", "by_source": { "mailroom/about_airline": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_banking": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_education": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_electricity": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/about_telecom": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/category": { "evaluated": 51, "successful_responses": 51, "errors": 0, "hard_targets": 51, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 51, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/from_billing_entity": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 79, "hard_accuracy_including_errors": 0.9080459770114943, "soft_targets": 0, "expected_accuracy_including_errors": 0.9080459770114943 }, "mailroom/has_billing_identifiers": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/is_promotional": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/kind": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 }, "mailroom/states_amount_owed": { "evaluated": 87, "successful_responses": 87, "errors": 0, "hard_targets": 87, "usable_decisions_with_probability_mass_failure": 0, "hard_correct": 87, "hard_accuracy_including_errors": 1.0, "soft_targets": 0, "expected_accuracy_including_errors": 1.0 } } } } } ], "trec": { "benchmarks": [ { "id": "dl19", "label": "TREC-DL19", "queries": 43, "downloaded_bm25_ndcg_at_10": 0.5058310024399073 }, { "id": "dl20", "label": "TREC-DL20", "queries": 54, "downloaded_bm25_ndcg_at_10": 0.4796366724252675 } ], "results": { "2b": { "status": "pending" }, "9b": { "status": "pending" }, "jev": { "status": "complete", "benchmarks": { "dl19": { "primary_strict_ndcg_at_10": 0.27583611543892894, "supplemental_actual_scalar_ndcg_at_10": 0.7282184260107177, "strict_complete_queries": 15, "scalar_complete_queries": 43 }, "dl20": { "primary_strict_ndcg_at_10": 0.1906665802224159, "supplemental_actual_scalar_ndcg_at_10": 0.7157341301273342, "strict_complete_queries": 16, "scalar_complete_queries": 54 } }, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/ir-control-v1/trec-holdout/jev-listwise-summary.json" }, "luna": { "status": "complete", "benchmarks": { "dl19": { "primary_strict_ndcg_at_10": 0.7299109599130584, "supplemental_actual_scalar_ndcg_at_10": null, "strict_complete_queries": 43, "scalar_complete_queries": null }, "dl20": { "primary_strict_ndcg_at_10": 0.7020823370921255, "supplemental_actual_scalar_ndcg_at_10": null, "strict_complete_queries": 54, "scalar_complete_queries": null } }, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/ir-control-v1/trec-holdout/luna-listwise-summary.json" }, "astra": { "status": "complete", "benchmarks": { "dl19": { "primary_strict_ndcg_at_10": 0.7366099323007737, "supplemental_actual_scalar_ndcg_at_10": null, "strict_complete_queries": 43, "scalar_complete_queries": null }, "dl20": { "primary_strict_ndcg_at_10": 0.7144837978397526, "supplemental_actual_scalar_ndcg_at_10": null, "strict_complete_queries": 54, "scalar_complete_queries": null } }, "evidence_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/ir-control-v1/trec-holdout/astra-listwise-summary.json" } }, "source_report_sha256": { "reports/ir-control-v1/trec-holdout/jev-listwise-summary.json": "2a4925b265c18cb1bad192cf96d824e96d2c4510eee259e80c95d8fb7501c20a", "reports/ir-control-v1/trec-holdout/luna-listwise-summary.json": "c3fdd38e5566a9593713be0e102953007c7ab6f551201dc380ca1f4d7d839684", "reports/ir-control-v1/trec-holdout/astra-listwise-summary.json": "b14bdc66ccc1a940de1a942fa07ca61134fab3f11718a942394d1522a93937f8" }, "method_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/reports/ir-control-v1/trec-holdout/README.md", "note": "nDCG@10 on all 97 judged queries with full official qrels and direct relevance grades. Each query reranks the downloaded BM25 top 100 using nine adaptive 20-passage Score windows. Strict failures contribute zero. Jev’s predeclared supplementary analysis uses actual scalar scores from mass-only failures, without renormalizing probabilities. Pending cells have no published model score. This independently authored protocol is not an exact reproduction of the community demo." }, "scope": "Fixed coverage checks and separate probes. Counts match the frozen supplied labels; a post-hoc audit flags six game references and one customer rubric ambiguity. Common-case sensitivity counts with identical exclusions are available in the method; primary counts stay unchanged. The 73,333-row full test/OOD registry remains pending; an additive 107,922-row registry is prepared but not evaluated. These are categorical decisions, not calibrated probabilities or closed-loop game success rates. Failed decisions count as incorrect; unattempted decisions remain pending.", "decision_policy": "Choice uses the returned selection. Noul uses argmax of [1-p, p], with exact ties selecting false. Score uses its most probable level; OpenAI returns a categorical integer. Jev probability-mass failures with usable finite choices are reported separately in decision-only analysis; vectors are never renormalized.", "method_url": "https://github.com/Zefan-Cai/Open-Jev/blob/main/docs/provider-comparison.md", "source_report_sha256": { "reports/provider-comparison-20260920/2b-coverage-quality-archive.json": "ac03294438cc2fe85cc03bcd5304d78cbe5291ea461b9ee21d5cd7b128ad9a88", "reports/provider-comparison-20260920/9b-coverage-quality-archive.json": "46e7d47846a75ebead59c7376e09db4fa97e2d2ab0f26c458d7b67a3d91ddb7d", "reports/provider-comparison-20260920/jev-coverage-quality.json": "f08013d9d189cdac5c712db4537cae301215210adcc466b23cb2f6592a6ac0a9", "reports/provider-comparison-20260920/openai-gpt-5.6-luna-coverage-quality.json": "f58d4239c04eb059d0463bab2b0e6442da1e5a487dde8fbc68e4dd1496aa2058", "reports/provider-comparison-20260920/openai-gpt-6-astra-coverage-quality.json": "7da5f1494ca1d7e8b139f7911379fd4dab0d6a665d3e1ebc81de6cc38564dd8d", "reports/provider-comparison-20260920/jev-jf100-quality.json": "19c6d776ae9cee2ced4e6ae46776bc403191872d6889e741673c6b9eef903974", "reports/provider-comparison-20260920/openai-gpt-5.6-luna-jf100-quality.json": "bcb634b0a7c39573fe47e969ca3360b03502cf6920b3ba37a7b1407b6b19fc48", "reports/provider-comparison-20260920/openai-gpt-6-astra-jf100-quality.json": "55ce041d4b8a21cea69554842dd7a603ac4241ccfb2435a89b84d2adb6f1a25b", "reports/provider-comparison-20260920/jev-ir-pilot-quality.json": "343dfcd968d69ceb63c0f4a800e6cc1d78f0db4a4ddd971b497a5c91bca087ea", "reports/provider-comparison-20260920/openai-gpt-5.6-luna-ir-pilot-quality.json": "90ff99921f0008df7e6895c1f2c034e05bfb444e16d15b5be7c5da80be16c8c5", "reports/provider-comparison-20260920/openai-gpt-6-astra-ir-pilot-quality.json": "9dbc87a87986c9cf3a0aefb6921123cb617c549bf5d2e9d0fc2ae8fe8c34a04d", "reports/provider-comparison-20260920/jev-fizzbuzz-quality.json": "bed8c9f7514e59a374ca55000721fc69b157ad1d7c19406fcdfda34aa4f80d65", "reports/provider-comparison-20260920/openai-gpt-5.6-luna-fizzbuzz-quality.json": "122ec66566611bb4337915003e43f2fa8bdfea82950b6cab723e3f04fe078508", "reports/provider-comparison-20260920/openai-gpt-6-astra-fizzbuzz-quality.json": "c3e50e9f2ccf6dfa63a0f25578a1142d5ff6e937cc0f9513d4c532194efa28d3", "reports/provider-comparison-20260920/jev-mailroom-quality.json": "272db97cf9e635ab4438c80542ee3d8a66fbcc1498e7e8260c512dda453de5b7", "reports/provider-comparison-20260920/openai-gpt-5.6-luna-mailroom-quality.json": "7b276d78d6da081d659b96a355a4174ab0f5581809d4c39d5edf55de16a0091e", "reports/provider-comparison-20260920/openai-gpt-6-astra-mailroom-quality.json": "20e08744273ea110897eebd3fcc16e1a825556cb3bafacb3cf2165435973962e" }, "public_data_policy": "Only derived results and hashes are published here. Original Wikispeedia text in request and raw-response bundles is excluded." }