{
  "scope": "Completed release-v2 checkpoint evaluation; broader prepared inventory is separate.",
  "evaluation_models": [
    {
      "model": "Qwen/Qwen3.5-2B",
      "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "evaluation_source_commit": "99c5361d83edd5f9dbd11cb3f4bd9f9c54b5973e",
      "audit_path": "reports/full-data-eval-n1-v1/2b-audit.json",
      "audit_sha256": "e8b0583b12ee00e8031e7ea5388d935389c3ce9681e3099da0cafe30329ff472",
      "coverage": {
        "expected_rows": 26452,
        "unique_ids": 26452,
        "shard_rows": [
          6613,
          6613,
          6613,
          6613
        ],
        "missing": 0,
        "duplicate": 0,
        "failed": 0,
        "merged_original_order_verified": true
      },
      "splits": {
        "test": {
          "overall": {
            "count": 10532,
            "hard_count": 10046,
            "accuracy": 0.9471431415488751,
            "expected_accuracy": 0.9195253563797051,
            "nll": 0.19538003409263777,
            "brier": 0.07611697556114595,
            "expected_brier": 0.10416793987302937,
            "multiclass_ece": 0.011605590146725192,
            "ece_bins": 15,
            "failed": 0,
            "hard_correct": 9515
          },
          "by_source": {
            "customer-control-v1": {
              "count": 606,
              "hard_count": 574,
              "accuracy": 1.0,
              "expected_accuracy": 0.9722222222222222,
              "nll": 0.04670156852740247,
              "brier": 0.0013588176050983829,
              "expected_brier": 0.029136595382876163,
              "multiclass_ece": 0.005803372421616342,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 574
            },
            "painting-geometry-v1": {
              "count": 3072,
              "hard_count": 3072,
              "accuracy": 0.9964192708333334,
              "expected_accuracy": 0.9964192708333334,
              "nll": 0.012696533729082774,
              "brier": 0.005503376394520538,
              "expected_brier": 0.0055033763945205386,
              "multiclass_ece": 0.0037695271810740086,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 3061
            },
            "reasoning-control-v1": {
              "count": 681,
              "hard_count": 681,
              "accuracy": 0.7797356828193832,
              "expected_accuracy": 0.7797356828193832,
              "nll": 0.5276862396629047,
              "brier": 0.30043397912430997,
              "expected_brier": 0.30043397912430997,
              "multiclass_ece": 0.06116577188210845,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 531
            },
            "snake-v1": {
              "count": 1392,
              "hard_count": 1279,
              "accuracy": 0.8780297107114934,
              "expected_accuracy": 0.8461446360153256,
              "nll": 0.3775572072892607,
              "brier": 0.18597837736131362,
              "expected_brier": 0.22704638502414884,
              "multiclass_ece": 0.0526435483183463,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 1123
            },
            "tic_tac_toe-v1": {
              "count": 472,
              "hard_count": 256,
              "accuracy": 0.4765625,
              "expected_accuracy": 0.41646489104116224,
              "nll": 1.1856874570601634,
              "brier": 0.3986960943836407,
              "expected_brier": 0.6686718813085801,
              "multiclass_ece": 0.05099574700428666,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 122
            },
            "tile_platformer-v1": {
              "count": 202,
              "hard_count": 202,
              "accuracy": 0.9752475247524752,
              "expected_accuracy": 0.9752475247524752,
              "nll": 0.13723787641738036,
              "brier": 0.048090580988872436,
              "expected_brier": 0.048090580988872436,
              "multiclass_ece": 0.01462437849252733,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 197
            },
            "trex_runner-v1": {
              "count": 4,
              "hard_count": 4,
              "accuracy": 0.25,
              "expected_accuracy": 0.25,
              "nll": 1.380181719997761,
              "brier": 0.8197811047574941,
              "expected_brier": 0.8197811047574941,
              "multiclass_ece": 0.47413478912915286,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 1
            },
            "vizdoom-basic-v1": {
              "count": 858,
              "hard_count": 858,
              "accuracy": 0.9685314685314685,
              "expected_accuracy": 0.9685314685314685,
              "nll": 0.08747268287742492,
              "brier": 0.04462544178795084,
              "expected_brier": 0.04462544178795084,
              "multiclass_ece": 0.0214576175261959,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 831
            },
            "wikispeedia-v1": {
              "count": 176,
              "hard_count": 51,
              "accuracy": 0.39215686274509803,
              "expected_accuracy": 0.25494105011150464,
              "nll": 2.2057266068313712,
              "brier": 0.32864256792900676,
              "expected_brier": 0.862757253748238,
              "multiclass_ece": 0.09196286653058698,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 20
            },
            "workflow-controls-v1/agent_trace_observability": {
              "count": 357,
              "hard_count": 357,
              "accuracy": 0.988795518207283,
              "expected_accuracy": 0.988795518207283,
              "nll": 0.026771587416750012,
              "brier": 0.0157428306754681,
              "expected_brier": 0.0157428306754681,
              "multiclass_ece": 0.006613012775607079,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 353
            },
            "workflow-controls-v1/customer_service": {
              "count": 456,
              "hard_count": 456,
              "accuracy": 0.9912280701754386,
              "expected_accuracy": 0.9912280701754386,
              "nll": 0.024863508782455843,
              "brier": 0.012699999145872375,
              "expected_brier": 0.012699999145872375,
              "multiclass_ece": 0.004145846144954706,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 452
            },
            "workflow-controls-v1/invoice_processing": {
              "count": 930,
              "hard_count": 930,
              "accuracy": 0.9989247311827957,
              "expected_accuracy": 0.9989247311827957,
              "nll": 0.010428173350270523,
              "brier": 0.0031761278683533505,
              "expected_brier": 0.0031761278683533475,
              "multiclass_ece": 0.009015233488700234,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 929
            },
            "workflow-controls-v1/security_incidents": {
              "count": 1326,
              "hard_count": 1326,
              "accuracy": 0.9962292609351433,
              "expected_accuracy": 0.9962292609351433,
              "nll": 0.014165561745883562,
              "brier": 0.0066243147587275175,
              "expected_brier": 0.006624314758727516,
              "multiclass_ece": 0.001806122714016116,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 1321
            }
          }
        },
        "ood": {
          "overall": {
            "count": 15920,
            "hard_count": 15446,
            "accuracy": 0.8602227113815875,
            "expected_accuracy": 0.8461552917002038,
            "nll": 0.8029899145502479,
            "brier": 0.24645039628025225,
            "expected_brier": 0.26378113592726077,
            "multiclass_ece": 0.10559429485648715,
            "ece_bins": 15,
            "failed": 0,
            "hard_correct": 13287
          },
          "by_source": {
            "customer-control-v1": {
              "count": 600,
              "hard_count": 560,
              "accuracy": 0.9732142857142857,
              "expected_accuracy": 0.9383333333333334,
              "nll": 0.18931729447777135,
              "brier": 0.053790307653173525,
              "expected_brier": 0.09045697431984019,
              "multiclass_ece": 0.02661442402817384,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 545
            },
            "painting-geometry-v1": {
              "count": 6912,
              "hard_count": 6912,
              "accuracy": 0.8051215277777778,
              "expected_accuracy": 0.8051215277777778,
              "nll": 1.4325737213947278,
              "brier": 0.3808534813920613,
              "expected_brier": 0.3808534813920613,
              "multiclass_ece": 0.18400414817622343,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 5565
            },
            "reasoning-control-v1": {
              "count": 750,
              "hard_count": 750,
              "accuracy": 0.6693333333333333,
              "expected_accuracy": 0.6693333333333333,
              "nll": 0.8174632190648329,
              "brier": 0.4497004885456776,
              "expected_brier": 0.4497004885456775,
              "multiclass_ece": 0.12814111983806595,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 502
            },
            "snake-v1": {
              "count": 3464,
              "hard_count": 3192,
              "accuracy": 0.868421052631579,
              "expected_accuracy": 0.8391070053887606,
              "nll": 0.40090561564728155,
              "brier": 0.20086908723576663,
              "expected_brier": 0.24022628508026242,
              "multiclass_ece": 0.07383729789301127,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 2772
            },
            "vizdoom-basic-v1": {
              "count": 825,
              "hard_count": 825,
              "accuracy": 0.9309090909090909,
              "expected_accuracy": 0.9309090909090909,
              "nll": 0.18737228735728542,
              "brier": 0.09844207618854224,
              "expected_brier": 0.09844207618854224,
              "multiclass_ece": 0.0250802098487241,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 768
            },
            "wikispeedia-v1": {
              "count": 219,
              "hard_count": 57,
              "accuracy": 0.5789473684210527,
              "expected_accuracy": 0.29281085479715613,
              "nll": 2.1593282501811903,
              "brier": 0.3142609630528501,
              "expected_brier": 0.8511196016238175,
              "multiclass_ece": 0.1249962258708074,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 33
            },
            "workflow-controls-v1/agent_trace_observability": {
              "count": 525,
              "hard_count": 525,
              "accuracy": 0.9885714285714285,
              "expected_accuracy": 0.9885714285714285,
              "nll": 0.032287053288972606,
              "brier": 0.01822595812319716,
              "expected_brier": 0.01822595812319716,
              "multiclass_ece": 0.005162135252793167,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 519
            },
            "workflow-controls-v1/customer_service": {
              "count": 600,
              "hard_count": 600,
              "accuracy": 0.9466666666666667,
              "expected_accuracy": 0.9466666666666667,
              "nll": 0.14410110761968423,
              "brier": 0.08122778878087651,
              "expected_brier": 0.08122778878087651,
              "multiclass_ece": 0.037479888817243255,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 568
            },
            "workflow-controls-v1/invoice_processing": {
              "count": 750,
              "hard_count": 750,
              "accuracy": 0.9973333333333333,
              "expected_accuracy": 0.9973333333333333,
              "nll": 0.013733993789974948,
              "brier": 0.005134315183105472,
              "expected_brier": 0.005134315183105474,
              "multiclass_ece": 0.008916569978317768,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 748
            },
            "workflow-controls-v1/security_incidents": {
              "count": 1275,
              "hard_count": 1275,
              "accuracy": 0.9937254901960785,
              "expected_accuracy": 0.9937254901960785,
              "nll": 0.01963776236994051,
              "brier": 0.010570254728764788,
              "expected_brier": 0.010570254728764786,
              "multiclass_ece": 0.002970435231356543,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 1267
            }
          }
        }
      }
    },
    {
      "model": "Qwen/Qwen3.5-9B",
      "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
      "evaluation_source_commit": "99c5361d83edd5f9dbd11cb3f4bd9f9c54b5973e",
      "audit_path": "reports/full-data-eval-n1-v1/9b-audit.json",
      "audit_sha256": "d490dd1636acebaf393fe181dfd0f65e66eb54edb7247b4abffd74a3d0891100",
      "coverage": {
        "expected_rows": 26452,
        "unique_ids": 26452,
        "shard_rows": [
          6613,
          6613,
          6613,
          6613
        ],
        "missing": 0,
        "duplicate": 0,
        "failed": 0,
        "merged_original_order_verified": true
      },
      "splits": {
        "test": {
          "overall": {
            "count": 10532,
            "hard_count": 10046,
            "accuracy": 0.9754130997411905,
            "expected_accuracy": 0.9471531754526437,
            "nll": 0.1309471525831089,
            "brier": 0.03911290032779779,
            "expected_brier": 0.0671638646396812,
            "multiclass_ece": 0.007706544094318023,
            "ece_bins": 15,
            "failed": 0,
            "hard_correct": 9799
          },
          "by_source": {
            "customer-control-v1": {
              "count": 606,
              "hard_count": 574,
              "accuracy": 1.0,
              "expected_accuracy": 0.9722222222222222,
              "nll": 0.04199953432734906,
              "brier": 6.843414604449945e-05,
              "expected_brier": 0.027846211923822282,
              "multiclass_ece": 0.0019262574800517217,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 574
            },
            "painting-geometry-v1": {
              "count": 3072,
              "hard_count": 3072,
              "accuracy": 1.0,
              "expected_accuracy": 1.0,
              "nll": 0.0013689140515597267,
              "brier": 7.957005332993052e-05,
              "expected_brier": 7.957005332993308e-05,
              "multiclass_ece": 0.0013415313830535288,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 3072
            },
            "reasoning-control-v1": {
              "count": 681,
              "hard_count": 681,
              "accuracy": 0.8722466960352423,
              "expected_accuracy": 0.8722466960352423,
              "nll": 0.30288131001112933,
              "brier": 0.1731250583140691,
              "expected_brier": 0.1731250583140691,
              "multiclass_ece": 0.0369016945577258,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 594
            },
            "snake-v1": {
              "count": 1392,
              "hard_count": 1279,
              "accuracy": 0.9976544175136826,
              "expected_accuracy": 0.9567768199233716,
              "nll": 0.08536951007564594,
              "brier": 0.01017585055261676,
              "expected_brier": 0.05124385821545201,
              "multiclass_ece": 0.014056450259188788,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 1276
            },
            "tic_tac_toe-v1": {
              "count": 472,
              "hard_count": 256,
              "accuracy": 0.64453125,
              "expected_accuracy": 0.5206315577078289,
              "nll": 1.0984003514512244,
              "brier": 0.3480926885371523,
              "expected_brier": 0.6180684754620918,
              "multiclass_ece": 0.06619460047533682,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 165
            },
            "tile_platformer-v1": {
              "count": 202,
              "hard_count": 202,
              "accuracy": 0.9752475247524752,
              "expected_accuracy": 0.9752475247524752,
              "nll": 0.13968696905117425,
              "brier": 0.047840610560813734,
              "expected_brier": 0.047840610560813734,
              "multiclass_ece": 0.035747153706735625,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 197
            },
            "trex_runner-v1": {
              "count": 4,
              "hard_count": 4,
              "accuracy": 0.75,
              "expected_accuracy": 0.75,
              "nll": 0.7668877262100418,
              "brier": 0.4059017097317489,
              "expected_brier": 0.40590170973174894,
              "multiclass_ece": 0.2272462205045263,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 3
            },
            "vizdoom-basic-v1": {
              "count": 858,
              "hard_count": 858,
              "accuracy": 0.9766899766899767,
              "expected_accuracy": 0.9766899766899767,
              "nll": 0.06761322089895994,
              "brier": 0.034824071109677196,
              "expected_brier": 0.034824071109677196,
              "multiclass_ece": 0.012718603010498264,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 838
            },
            "wikispeedia-v1": {
              "count": 176,
              "hard_count": 51,
              "accuracy": 0.45098039215686275,
              "expected_accuracy": 0.2709042535747081,
              "nll": 2.1680481914874337,
              "brier": 0.31628613125660077,
              "expected_brier": 0.8504008170758321,
              "multiclass_ece": 0.0965942356793526,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 23
            },
            "workflow-controls-v1/agent_trace_observability": {
              "count": 357,
              "hard_count": 357,
              "accuracy": 1.0,
              "expected_accuracy": 1.0,
              "nll": 0.00044687482748441643,
              "brier": 2.946417322061725e-06,
              "expected_brier": 2.946417322060909e-06,
              "multiclass_ece": 0.0004461355302124166,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 357
            },
            "workflow-controls-v1/customer_service": {
              "count": 456,
              "hard_count": 456,
              "accuracy": 1.0,
              "expected_accuracy": 1.0,
              "nll": 0.0004573076748327324,
              "brier": 8.826622033700542e-06,
              "expected_brier": 8.826622033703088e-06,
              "multiclass_ece": 0.0004550441009002813,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 456
            },
            "workflow-controls-v1/invoice_processing": {
              "count": 930,
              "hard_count": 930,
              "accuracy": 0.9967741935483871,
              "expected_accuracy": 0.9967741935483871,
              "nll": 0.005458157001187219,
              "brier": 0.003648645648216325,
              "expected_brier": 0.0036486456482163225,
              "multiclass_ece": 0.0037258424516831987,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 927
            },
            "workflow-controls-v1/security_incidents": {
              "count": 1326,
              "hard_count": 1326,
              "accuracy": 0.9932126696832579,
              "expected_accuracy": 0.9932126696832579,
              "nll": 0.022337704900579736,
              "brier": 0.011355443994287867,
              "expected_brier": 0.011355443994287865,
              "multiclass_ece": 0.004366133269192267,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 1317
            }
          }
        },
        "ood": {
          "overall": {
            "count": 15920,
            "hard_count": 15446,
            "accuracy": 0.9196555742587078,
            "expected_accuracy": 0.9040437668138673,
            "nll": 0.2994407995801986,
            "brier": 0.1266465263060649,
            "expected_brier": 0.1439772659530734,
            "multiclass_ece": 0.037398104632606045,
            "ece_bins": 15,
            "failed": 0,
            "hard_correct": 14205
          },
          "by_source": {
            "customer-control-v1": {
              "count": 600,
              "hard_count": 560,
              "accuracy": 1.0,
              "expected_accuracy": 0.9633333333333334,
              "nll": 0.06367998388489432,
              "brier": 0.002432497344975918,
              "expected_brier": 0.03909916401164259,
              "multiclass_ece": 0.007595238345599102,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 560
            },
            "painting-geometry-v1": {
              "count": 6912,
              "hard_count": 6912,
              "accuracy": 0.8629918981481481,
              "expected_accuracy": 0.8629918981481481,
              "nll": 0.4601822966946938,
              "brier": 0.22129407318457964,
              "expected_brier": 0.22129407318457964,
              "multiclass_ece": 0.0793900158186277,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 5965
            },
            "reasoning-control-v1": {
              "count": 750,
              "hard_count": 750,
              "accuracy": 0.736,
              "expected_accuracy": 0.736,
              "nll": 0.6801700551809211,
              "brier": 0.3601332438976597,
              "expected_brier": 0.3601332438976597,
              "multiclass_ece": 0.0941763492512841,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 552
            },
            "snake-v1": {
              "count": 3464,
              "hard_count": 3192,
              "accuracy": 0.9899749373433584,
              "expected_accuracy": 0.9514049268668205,
              "nll": 0.10915249045149827,
              "brier": 0.020795628523113556,
              "expected_brier": 0.06015282636760932,
              "multiclass_ece": 0.013368579791205319,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 3160
            },
            "vizdoom-basic-v1": {
              "count": 825,
              "hard_count": 825,
              "accuracy": 0.9745454545454545,
              "expected_accuracy": 0.9745454545454545,
              "nll": 0.07717523550004791,
              "brier": 0.0398977614679421,
              "expected_brier": 0.0398977614679421,
              "multiclass_ece": 0.021055164766154082,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 804
            },
            "wikispeedia-v1": {
              "count": 219,
              "hard_count": 57,
              "accuracy": 0.6666666666666666,
              "expected_accuracy": 0.3274433836077672,
              "nll": 2.111795103417709,
              "brier": 0.29905520762617627,
              "expected_brier": 0.8359138461971435,
              "multiclass_ece": 0.14396741883440964,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 38
            },
            "workflow-controls-v1/agent_trace_observability": {
              "count": 525,
              "hard_count": 525,
              "accuracy": 1.0,
              "expected_accuracy": 1.0,
              "nll": 0.000545160068456591,
              "brier": 4.895507169664063e-06,
              "expected_brier": 4.8955071696642534e-06,
              "multiclass_ece": 0.0005439303275837422,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 525
            },
            "workflow-controls-v1/customer_service": {
              "count": 600,
              "hard_count": 600,
              "accuracy": 0.9783333333333334,
              "expected_accuracy": 0.9783333333333334,
              "nll": 0.16283998438208533,
              "brier": 0.04328014604238859,
              "expected_brier": 0.04328014604238859,
              "multiclass_ece": 0.021130186926374108,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 587
            },
            "workflow-controls-v1/invoice_processing": {
              "count": 750,
              "hard_count": 750,
              "accuracy": 0.9973333333333333,
              "expected_accuracy": 0.9973333333333333,
              "nll": 0.006416542163341697,
              "brier": 0.003983146029347859,
              "expected_brier": 0.003983146029347861,
              "multiclass_ece": 0.002603556053682448,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 748
            },
            "workflow-controls-v1/security_incidents": {
              "count": 1275,
              "hard_count": 1275,
              "accuracy": 0.9929411764705882,
              "expected_accuracy": 0.9929411764705882,
              "nll": 0.02425288839114142,
              "brier": 0.012286914067882482,
              "expected_brier": 0.012286914067882486,
              "multiclass_ece": 0.004988858941316103,
              "ece_bins": 15,
              "failed": 0,
              "hard_correct": 1266
            }
          }
        }
      }
    }
  ],
  "prepared_inventory": {
    "corpora": [
      {
        "name": "browser-drone-expansion-v1",
        "splits": {
          "calibration": 6883,
          "ood": 25379,
          "test": 14902,
          "train": 110324,
          "validation": 5562
        },
        "manifest_path": "reports/data-manifests/browser-drone-expansion-v1.json",
        "manifest_sha256": "ba001b88787ce21576017897a0cbdedd5917ceb29ad3522b0e1fb7b4836d49df"
      },
      {
        "name": "citation-control-v1",
        "splits": {
          "calibration": 200,
          "ood": 800,
          "test": 300,
          "train": 2520,
          "validation": 180
        },
        "manifest_path": "reports/data-manifests/citation-control-v1.json",
        "manifest_sha256": "069466d7536c579812bf2c870a9ea209d13a695005c299debffe61cd3aeffff4"
      },
      {
        "name": "entity-alignment-control-v1",
        "splits": {
          "calibration": 728,
          "ood": 2240,
          "test": 1008,
          "train": 6944,
          "validation": 280
        },
        "manifest_path": "reports/data-manifests/entity-alignment-control-v1.json",
        "manifest_sha256": "b3afddbeb926d57b7d7ea13c5fbda0fff878c647ea9ff1ee8a09b11d7f2f9eb5"
      },
      {
        "name": "amount-extraction-control-v1",
        "splits": {
          "calibration": 2232,
          "ood": 9920,
          "test": 3472,
          "train": 32984,
          "validation": 992
        },
        "manifest_path": "reports/data-manifests/amount-extraction-control-v1.json",
        "manifest_sha256": "6f89f81336ac36505c563193da27fca2155fa566175451854f0ac410979ea80b"
      },
      {
        "name": "email-selection-control-v1",
        "splits": {
          "calibration": 81,
          "ood": 1080,
          "test": 432,
          "train": 3618,
          "validation": 189
        },
        "manifest_path": "reports/data-manifests/email-selection-control-v1.json",
        "manifest_sha256": "4c18cc791425003c5c8f052ce27ed034dfb54c79ddac24de13f7290ab3b3316c"
      },
      {
        "name": "phone-extraction-control-v1",
        "splits": {
          "calibration": 855,
          "ood": 3800,
          "test": 1615,
          "train": 12350,
          "validation": 380
        },
        "manifest_path": "reports/data-manifests/phone-extraction-control-v1.json",
        "manifest_sha256": "1001f1e04795308bf4f68db51076e8e1e1e47802986de54fc33b3b8d33548cfb"
      },
      {
        "name": "context-retention-control-v1",
        "splits": {
          "calibration": 456,
          "ood": 2160,
          "test": 522,
          "train": 6138,
          "validation": 558
        },
        "manifest_path": "reports/context-retention-control-v1/manifest.json",
        "manifest_sha256": "03badf69899809919e90bd299c5ddea4b3f8d646505d02f9544a1312e13bbd94"
      },
      {
        "name": "sponsor-segment-control-v1",
        "splits": {
          "calibration": 756,
          "ood": 2160,
          "test": 999,
          "train": 6345,
          "validation": 540
        },
        "manifest_path": "reports/sponsor-segment-control-v1/manifest.json",
        "manifest_sha256": "6a143ede0485b22588fc92f4612f41428c46b1fa7d8f8cc0d32422d09b548f13"
      },
      {
        "name": "silent-failure-control-v1",
        "splits": {
          "calibration": 264,
          "ood": 1920,
          "test": 624,
          "train": 6432,
          "validation": 360
        },
        "manifest_path": "reports/silent-failure-control-v1/manifest.json",
        "manifest_sha256": "a7d326a480c0132dd608d0d7119a948bc16e36667e0ff7049e111f33c5445fac"
      },
      {
        "name": "ir-control-v1",
        "splits": {
          "calibration": 464,
          "ood": 2320,
          "test": 986,
          "train": 7366,
          "validation": 464
        },
        "manifest_path": "reports/data-manifests/ir-control-v1.json",
        "manifest_sha256": "5305dad7326fbf32aa006447137c5a3eb92872d90cde71302bc7efb39f882590"
      },
      {
        "name": "mailroom-control-v1",
        "splits": {
          "calibration": 4018,
          "ood": 22960,
          "test": 8323,
          "train": 73472,
          "validation": 6027
        },
        "manifest_path": "reports/data-manifests/mailroom-control-v1.json",
        "manifest_sha256": "afa9daabd5639a333c9adf8e89e7c7bbfcf84adfc051fb9843978c29703122f6"
      }
    ],
    "splits": {
      "calibration": 16937,
      "ood": 74739,
      "test": 33183,
      "train": 268493,
      "validation": 15532
    },
    "total_decision_rows": 408884,
    "task_source_identifiers": 25
  },
  "limitations": [
    "2B and 9B trained on 80,816 release-v2 training rows; broader inventory is not their evaluation set.",
    "Hard accuracy excludes 960 soft-target rows per model; expected accuracy includes all rows.",
    "Synthetic controlled decision accuracy is not an end-to-end task or gameplay success rate.",
    "No full-data base-model baseline was run; no full-data training gain is claimed.",
    "27B final evaluation and final-model JF100/service suites are pending.",
    "The new corpora have not been used to retrain the released models; new-domain Open-Jev evaluation remains pending.",
    "JF100 is a separate holdout: 100 questions with three option rotations."
  ]
}