{"version":"consensus-calibration-2026-09-21","generatedAt":"2026-09-21T20:02:25.169Z","snapshotId":"v0-fixture","inputHash":"sha256-0ac31ab8b72c476d95e4ab521470c30769913640e8d9723352206beb310cfdf8","excludedModels":["grok-4.7"],"scope":"Reused-data method selection, not untouched prospective validation. Entire sources/families are withheld. Missing predictions remain uncovered; targets are weighted with the same comparison budgets for every candidate. Fold means give each source/family equal diagnostic weight. Scores are not probabilities of task success.","selectionRule":"Minimize source-held-out mean log loss among candidates that do not worsen source/family log loss, Brier score, or coverage versus the baseline. Direct-order reversals and source sensitivity are separately reported, not optimized to an expected model order.","baseline":0.2,"selected":0.1,"candidates":[0.05,0.1,0.2,0.4],"summary":{"source":{"0.05":{"folds":11,"predictableFolds":11,"loss":0.4469114587142892,"brier":0.13884651729860728,"accuracy":0.7799093084486973,"coverage":0.7876354262876734},"0.1":{"folds":11,"predictableFolds":11,"loss":0.443637626718917,"brier":0.13890769511870346,"accuracy":0.778519448544144,"coverage":0.7876354262876734},"0.2":{"folds":11,"predictableFolds":11,"loss":0.45923278685438973,"brier":0.14414407442546934,"accuracy":0.7756704199007992,"coverage":0.7876354262876734},"0.4":{"folds":11,"predictableFolds":11,"loss":0.5094464975977433,"brier":0.16220884650884804,"accuracy":0.7830875039420413,"coverage":0.7876354262876734}},"family":{"0.05":{"folds":49,"predictableFolds":48,"loss":0.40837616080902067,"brier":0.12438434118580617,"accuracy":0.8186929069809187,"coverage":0.9669487730799103},"0.1":{"folds":49,"predictableFolds":48,"loss":0.4107635332912709,"brier":0.1255164667133952,"accuracy":0.8179533901764359,"coverage":0.9669487730799103},"0.2":{"folds":49,"predictableFolds":48,"loss":0.43086889668901623,"brier":0.13133028159815088,"accuracy":0.8169599829761948,"coverage":0.9669487730799103},"0.4":{"folds":49,"predictableFolds":48,"loss":0.48683442280955447,"brier":0.1500478270467948,"accuracy":0.8065174486714416,"coverage":0.9669487730799103}}},"folds":[{"kind":"source","omitted":"Artificial Analysis","metrics":{"0.05":{"converged":true,"loss":0.40868474265887733,"brier":0.1246927102204791,"accuracy":0.8219513490311335,"coverage":0.07673637228064038},"0.1":{"converged":true,"loss":0.3925807848824492,"brier":0.12285053104608741,"accuracy":0.8210009387218301,"coverage":0.07673637228064038},"0.2":{"converged":true,"loss":0.3957868435058506,"brier":0.12435088181868935,"accuracy":0.8193583595398186,"coverage":0.07673637228064038},"0.4":{"converged":true,"loss":0.4412191895041359,"brier":0.13758901919576774,"accuracy":0.8182310934740961,"coverage":0.07673637228064038}}},{"kind":"source","omitted":"ARC Prize","metrics":{"0.05":{"converged":true,"loss":0.3808796991968827,"brier":0.11001182284291433,"accuracy":0.8520972792921395,"coverage":0.596428585126048},"0.1":{"converged":true,"loss":0.3716899274410811,"brier":0.11115627356439947,"accuracy":0.8510651979602722,"coverage":0.596428585126048},"0.2":{"converged":true,"loss":0.3856037252800743,"brier":0.11765803249339615,"accuracy":0.8477824719794734,"coverage":0.596428585126048},"0.4":{"converged":true,"loss":0.44289579074626756,"brier":0.13763238625350327,"accuracy":0.8422619621763083,"coverage":0.596428585126048}}},{"kind":"source","omitted":"Datacurve","metrics":{"0.05":{"converged":true,"loss":0.4135311483245137,"brier":0.13237745105842336,"accuracy":0.7964929214929215,"coverage":1},"0.1":{"converged":true,"loss":0.4295338525426164,"brier":0.13601820603787804,"accuracy":0.8023487773487776,"coverage":1},"0.2":{"converged":true,"loss":0.46435624415969096,"brier":0.1463920797616732,"accuracy":0.8033140283140288,"coverage":1},"0.4":{"converged":true,"loss":0.5282346291092486,"brier":0.17050068407003197,"accuracy":0.8019948519948522,"coverage":1}}},{"kind":"source","omitted":"LiveCodeBench","metrics":{"0.05":{"converged":true,"loss":0.4007594188452837,"brier":0.13832324006261318,"accuracy":0.6666666666666666,"coverage":1},"0.1":{"converged":true,"loss":0.4096681935671248,"brier":0.13406403185633103,"accuracy":0.6666666666666666,"coverage":1},"0.2":{"converged":true,"loss":0.44150765748426113,"brier":0.13869325146739314,"accuracy":0.6666666666666666,"coverage":1},"0.4":{"converged":true,"loss":0.5072861565667148,"brier":0.16167614559759447,"accuracy":0.6666666666666666,"coverage":1}}},{"kind":"source","omitted":"Scale","metrics":{"0.05":{"converged":true,"loss":0.5463030465126703,"brier":0.18751720600483993,"accuracy":0.6934306569343065,"coverage":0.6327944572748267},"0.1":{"converged":true,"loss":0.5326595315180346,"brier":0.18205673502369565,"accuracy":0.6934306569343065,"coverage":0.6327944572748267},"0.2":{"converged":true,"loss":0.5388029842314236,"brier":0.18205474682718142,"accuracy":0.6934306569343065,"coverage":0.6327944572748267},"0.4":{"converged":true,"loss":0.5765042902883791,"brier":0.1952612082338054,"accuracy":0.6934306569343065,"coverage":0.6327944572748267}}},{"kind":"source","omitted":"Terminal-Bench","metrics":{"0.05":{"converged":true,"loss":0.3349908737948209,"brier":0.09459200627184193,"accuracy":0.8394618834080719,"coverage":0.9139344262295086},"0.1":{"converged":true,"loss":0.3671387821943355,"brier":0.10455255744922548,"accuracy":0.8394618834080719,"coverage":0.9139344262295086},"0.2":{"converged":true,"loss":0.4209256254633584,"brier":0.1238334238815827,"accuracy":0.8179372197309418,"coverage":0.9139344262295086},"0.4":{"converged":true,"loss":0.5048868959492139,"brier":0.1590084906482406,"accuracy":0.925560538116592,"coverage":0.9139344262295086}}},{"kind":"source","omitted":"Cognition","metrics":{"0.05":{"converged":true,"loss":0.30972081961270215,"brier":0.09567090699357882,"accuracy":0.8643526350032384,"coverage":0.97562523803478},"0.1":{"converged":true,"loss":0.32880490862633477,"brier":0.09986030386582034,"accuracy":0.8601366297982938,"coverage":0.97562523803478},"0.2":{"converged":true,"loss":0.37103741402838786,"brier":0.1114095063800974,"accuracy":0.8552700065061666,"coverage":0.97562523803478},"0.4":{"converged":true,"loss":0.45323986842472025,"brier":0.1396310262959477,"accuracy":0.8537518976360734,"coverage":0.97562523803478}}},{"kind":"source","omitted":"LiveBench","metrics":{"0.05":{"converged":true,"loss":0.6985448469219426,"brier":0.1965079899691582,"accuracy":0.6836263736264058,"coverage":0.9259259259259489},"0.1":{"converged":true,"loss":0.6465378658231548,"brier":0.18872146504172882,"accuracy":0.6836263736264058,"coverage":0.9259259259259489},"0.2":{"converged":true,"loss":0.6043361346179026,"brier":0.18043237807887655,"accuracy":0.6836263736264058,"coverage":0.9259259259259489},"0.4":{"converged":true,"loss":0.5948085281106537,"brier":0.17852811567751053,"accuracy":0.6841758241758564,"coverage":0.9259259259259489}}},{"kind":"source","omitted":"SWE-rebench","metrics":{"0.05":{"converged":true,"loss":0.4315407721502176,"brier":0.13697349724801527,"accuracy":0.8139861927895675,"coverage":1},"0.1":{"converged":true,"loss":0.4507267626303278,"brier":0.14355508694548774,"accuracy":0.8032472513423675,"coverage":1},"0.2":{"converged":true,"loss":0.4863432142202282,"brier":0.15602188105044085,"accuracy":0.8032472513423675,"coverage":1},"0.4":{"converged":true,"loss":0.5445209594015045,"brier":0.17869059162445594,"accuracy":0.7844541038097672,"coverage":1}}},{"kind":"source","omitted":"Epoch AI","metrics":{"0.05":{"converged":true,"loss":0.47827180757873206,"brier":0.14301063653404808,"accuracy":0.7939403669523198,"coverage":0.6024964331150393},"0.1":{"converged":true,"loss":0.44655367402945595,"brier":0.1393893258377114,"accuracy":0.7940652638198092,"coverage":0.6024964331150393},"0.2":{"converged":true,"loss":0.43353829352725776,"brier":0.13764685807405105,"accuracy":0.7932756211188461,"coverage":0.6024964331150393},"0.4":{"converged":true,"loss":0.4654805855336791,"brier":0.14664012055311404,"accuracy":0.7967835335035227,"coverage":0.6024964331150393}}},{"kind":"source","omitted":"Vals AI","metrics":{"0.05":{"converged":true,"loss":0.512798870260538,"brier":0.16763422307876788,"accuracy":0.7529960677388999,"coverage":0.9400482511776134},"0.1":{"converged":true,"loss":0.5041196106531728,"brier":0.16576012963737244,"accuracy":0.7486642943587826,"coverage":0.9400482511776134},"0.2":{"converged":true,"loss":0.5093225188798509,"brier":0.16709177884678103,"accuracy":0.7484659631497718,"coverage":0.9400482511776134},"0.4":{"converged":true,"loss":0.5448345799406596,"brier":0.17913952344735704,"accuracy":0.746651414874413,"coverage":0.9400482511776134}}},{"kind":"family","omitted":"analyst-agent","metrics":{"0.05":{"converged":true,"loss":0.3095783974380786,"brier":0.09504150852513064,"accuracy":0.8596059113300478,"coverage":1},"0.1":{"converged":true,"loss":0.32503757572515396,"brier":0.09790981422413612,"accuracy":0.8596059113300479,"coverage":1},"0.2":{"converged":true,"loss":0.36398042914624235,"brier":0.10745286081521367,"accuracy":0.8620689655172399,"coverage":1},"0.4":{"converged":true,"loss":0.4457768061149453,"brier":0.13382533911441685,"accuracy":0.8522167487684714,"coverage":1}}},{"kind":"family","omitted":"briefcase","metrics":{"0.05":{"converged":true,"loss":0.3137823414600603,"brier":0.09666861226578873,"accuracy":0.857910435564289,"coverage":1},"0.1":{"converged":true,"loss":0.3191266623879598,"brier":0.09753240821136615,"accuracy":0.8580044128463028,"coverage":1},"0.2":{"converged":true,"loss":0.34634557933089116,"brier":0.10357338119160472,"accuracy":0.8591321402304658,"coverage":1},"0.4":{"converged":true,"loss":0.4195019653891971,"brier":0.1252939122700848,"accuracy":0.8592220315436968,"coverage":1}}},{"kind":"family","omitted":"aime","metrics":{"0.05":{"converged":true,"loss":0.35591354353583143,"brier":0.11194880287000677,"accuracy":0.8362659526997005,"coverage":1},"0.1":{"converged":true,"loss":0.3662582749551732,"brier":0.11413950543127174,"accuracy":0.8361338688085804,"coverage":1},"0.2":{"converged":true,"loss":0.3952455283182114,"brier":0.12161671883693957,"accuracy":0.8359723337795756,"coverage":1},"0.4":{"converged":true,"loss":0.4615791294775816,"brier":0.1432233031129244,"accuracy":0.8346041945560149,"coverage":1}}},{"kind":"family","omitted":"apex-agents","metrics":{"0.05":{"converged":true,"loss":0.3145761129411575,"brier":0.09458833897473912,"accuracy":0.868945868945869,"coverage":1},"0.1":{"converged":true,"loss":0.35022099669091006,"brier":0.10520834620144115,"accuracy":0.8660968660968662,"coverage":1},"0.2":{"converged":true,"loss":0.4092161914203821,"brier":0.12473768803904174,"accuracy":0.8461538461538463,"coverage":1},"0.4":{"converged":true,"loss":0.49798623907690437,"brier":0.15866623240945243,"accuracy":0.8176638176638178,"coverage":1}}},{"kind":"family","omitted":"aa-lcr","metrics":{"0.05":{"converged":true,"loss":0.2789274933254238,"brier":0.0823952331488644,"accuracy":0.8811610027164497,"coverage":1},"0.1":{"converged":true,"loss":0.2892812422747993,"brier":0.0846706585642385,"accuracy":0.881165549721193,"coverage":1},"0.2":{"converged":true,"loss":0.3208105555935885,"brier":0.09235481407138997,"accuracy":0.88118660433011,"coverage":1},"0.4":{"converged":true,"loss":0.3975259632670178,"brier":0.11556907591581841,"accuracy":0.8800560176608238,"coverage":1}}},{"kind":"family","omitted":"automationbench","metrics":{"0.05":{"converged":true,"loss":0.2568304824006806,"brier":0.07896829257123941,"accuracy":0.8842564102564112,"coverage":1},"0.1":{"converged":true,"loss":0.27088081543914955,"brier":0.08145545537348023,"accuracy":0.8863944773175552,"coverage":1},"0.2":{"converged":true,"loss":0.3082483300028534,"brier":0.09010520926807841,"accuracy":0.8877475345167662,"coverage":1},"0.4":{"converged":true,"loss":0.3928998464074472,"brier":0.11560438717433368,"accuracy":0.8867218934911252,"coverage":1}}},{"kind":"family","omitted":"critpt","metrics":{"0.05":{"converged":true,"loss":0.40402567708988496,"brier":0.07163771320153464,"accuracy":0.8223202822511261,"coverage":1},"0.1":{"converged":true,"loss":0.39289809612437354,"brier":0.07038413245436652,"accuracy":0.8217746337912754,"coverage":1},"0.2":{"converged":true,"loss":0.3974202484168919,"brier":0.0718897159600353,"accuracy":0.8203297647570102,"coverage":1},"0.4":{"converged":true,"loss":0.44391687978157873,"brier":0.08502203458177436,"accuracy":0.8187088678615639,"coverage":1}}},{"kind":"family","omitted":"enterprise-ops","metrics":{"0.05":{"converged":true,"loss":0.4175349615366484,"brier":0.1352905556265247,"accuracy":0.8033642691415349,"coverage":1},"0.1":{"converged":true,"loss":0.41585717240244957,"brier":0.13460911332424044,"accuracy":0.8016241299303981,"coverage":1},"0.2":{"converged":true,"loss":0.4355649742506771,"brier":0.13917682123017303,"accuracy":0.8004640371229734,"coverage":1},"0.4":{"converged":true,"loss":0.49526533444390825,"brier":0.1587842947754412,"accuracy":0.7998839907192612,"coverage":1}}},{"kind":"family","omitted":"gdp-pdf","metrics":{"0.05":{"converged":true,"loss":0.2803187821361007,"brier":0.08647406659883446,"accuracy":0.871402842679114,"coverage":1},"0.1":{"converged":true,"loss":0.2939480101574615,"brier":0.0891073282514396,"accuracy":0.8696529400311384,"coverage":1},"0.2":{"converged":true,"loss":0.330773718995391,"brier":0.09785935681120697,"accuracy":0.867559871495312,"coverage":1},"0.4":{"converged":true,"loss":0.41276675590185935,"brier":0.12293186191657036,"accuracy":0.8624123831775534,"coverage":1}}},{"kind":"family","omitted":"gdpval","metrics":{"0.05":{"converged":true,"loss":0.25039409831323145,"brier":0.07783887064202435,"accuracy":0.8891651360097831,"coverage":1},"0.1":{"converged":true,"loss":0.26521188750076424,"brier":0.08073684757170597,"accuracy":0.8885266334295328,"coverage":1},"0.2":{"converged":true,"loss":0.30270769645090256,"brier":0.089487763261731,"accuracy":0.8876432257500085,"coverage":1},"0.4":{"converged":true,"loss":0.38701812797506013,"brier":0.11455276990430097,"accuracy":0.8858632904749265,"coverage":1}}},{"kind":"family","omitted":"gmmlu","metrics":{"0.05":{"converged":true,"loss":0.3018894376241884,"brier":0.09301927828243292,"accuracy":0.8742977528089891,"coverage":1},"0.1":{"converged":true,"loss":0.32153731113493184,"brier":0.09751122807299861,"accuracy":0.8747659176029967,"coverage":1},"0.2":{"converged":true,"loss":0.3631080910512544,"brier":0.10878308800011087,"accuracy":0.8725031210986272,"coverage":1},"0.4":{"converged":true,"loss":0.44415542034850714,"brier":0.13604668830537983,"accuracy":0.8681335830212238,"coverage":1}}},{"kind":"family","omitted":"gpqa","metrics":{"0.05":{"converged":true,"loss":0.23750729453178546,"brier":0.06852884662733008,"accuracy":0.9134626676121257,"coverage":0.9926527318658573},"0.1":{"converged":true,"loss":0.2553090011198598,"brier":0.07301592676102377,"accuracy":0.9132698687087591,"coverage":0.9926527318658573},"0.2":{"converged":true,"loss":0.2953038965990647,"brier":0.08373760884726976,"accuracy":0.9125192930295873,"coverage":0.9926527318658573},"0.4":{"converged":true,"loss":0.380764265401413,"brier":0.11057369270352761,"accuracy":0.9114290234450321,"coverage":0.9926527318658573}}},{"kind":"family","omitted":"harvey","metrics":{"0.05":{"converged":true,"loss":0.5479936814635115,"brier":0.17301330122294498,"accuracy":0.7389725274725253,"coverage":1},"0.1":{"converged":true,"loss":0.5261541063429246,"brier":0.1688260843764184,"accuracy":0.7398846153846131,"coverage":1},"0.2":{"converged":true,"loss":0.5184392603784599,"brier":0.166969163668691,"accuracy":0.7409798534798512,"coverage":1},"0.4":{"converged":true,"loss":0.5445902536886883,"brier":0.17547912242273006,"accuracy":0.7464633699633682,"coverage":1}}},{"kind":"family","omitted":"hle","metrics":{"0.05":{"converged":true,"loss":0.25729393968193215,"brier":0.07814529257245076,"accuracy":0.8945690611930084,"coverage":0.9825473736189066},"0.1":{"converged":true,"loss":0.2718317349793689,"brier":0.08143405050506519,"accuracy":0.8940318109718265,"coverage":0.9825473736189066},"0.2":{"converged":true,"loss":0.3071719780509617,"brier":0.09019491467123622,"accuracy":0.8926314267168555,"coverage":0.9825473736189066},"0.4":{"converged":true,"loss":0.3873229873429865,"brier":0.1143555191327396,"accuracy":0.891178691648616,"coverage":0.9825473736189066}}},{"kind":"family","omitted":"ifbench","metrics":{"0.05":{"converged":true,"loss":0.40667265872924485,"brier":0.1256725951846165,"accuracy":0.8224168937329378,"coverage":1},"0.1":{"converged":true,"loss":0.3987754224571157,"brier":0.12527554815979136,"accuracy":0.8219891008174066,"coverage":1},"0.2":{"converged":true,"loss":0.40850031438244466,"brier":0.12856372722482515,"accuracy":0.8215985467756264,"coverage":1},"0.4":{"converged":true,"loss":0.4594978340139958,"brier":0.1446391079786159,"accuracy":0.8206248864668162,"coverage":1}}},{"kind":"family","omitted":"itbench","metrics":{"0.05":{"converged":true,"loss":0.3376658474265422,"brier":0.10819337645981526,"accuracy":0.8366093366093381,"coverage":1},"0.1":{"converged":true,"loss":0.35715478580183957,"brier":0.11216926023270568,"accuracy":0.8402948402948418,"coverage":1},"0.2":{"converged":true,"loss":0.4003837854269967,"brier":0.1239410947440089,"accuracy":0.8427518427518442,"coverage":1},"0.4":{"converged":true,"loss":0.4805982442461793,"brier":0.15194498414433072,"accuracy":0.839066339066341,"coverage":1}}},{"kind":"family","omitted":"livecodebench","metrics":{"0.05":{"converged":true,"loss":0.3232918108817062,"brier":0.09789406494580605,"accuracy":0.8709185132237405,"coverage":1},"0.1":{"converged":true,"loss":0.3438737797390919,"brier":0.10324057429874212,"accuracy":0.8703585894686774,"coverage":1},"0.2":{"converged":true,"loss":0.3849626738737683,"brier":0.11540108555190055,"accuracy":0.8685668334524753,"coverage":1},"0.4":{"converged":true,"loss":0.46224221334044424,"brier":0.14282519893797696,"accuracy":0.8653097450560019,"coverage":1}}},{"kind":"family","omitted":"math500","metrics":{"0.05":{"converged":true,"loss":0.3353690532291463,"brier":0.10206062061079518,"accuracy":0.8667452830188727,"coverage":1},"0.1":{"converged":true,"loss":0.3556338822168585,"brier":0.10746168582948475,"accuracy":0.866090146750529,"coverage":1},"0.2":{"converged":true,"loss":0.3960987114996431,"brier":0.11968113676531497,"accuracy":0.8674004192872168,"coverage":1},"0.4":{"converged":true,"loss":0.4721166359018896,"brier":0.14704789930987855,"accuracy":0.8679245283018918,"coverage":1}}},{"kind":"family","omitted":"mmlu-pro","metrics":{"0.05":{"converged":true,"loss":0.33024200324664144,"brier":0.10144170842039113,"accuracy":0.8626076907038394,"coverage":1},"0.1":{"converged":true,"loss":0.34908173626864053,"brier":0.10613134020283964,"accuracy":0.8618998600065527,"coverage":1},"0.2":{"converged":true,"loss":0.3878737386748441,"brier":0.11731933242692082,"accuracy":0.862281476186221,"coverage":1},"0.4":{"converged":true,"loss":0.46291261956102503,"brier":0.1436473918883971,"accuracy":0.8624236737854826,"coverage":1}}},{"kind":"family","omitted":"mmmu-pro","metrics":{"0.05":{"converged":true,"loss":0.23265912354188067,"brier":0.06883357515976066,"accuracy":0.9107090734718419,"coverage":1},"0.1":{"converged":true,"loss":0.25136978909787416,"brier":0.07296129587358999,"accuracy":0.910761195896994,"coverage":1},"0.2":{"converged":true,"loss":0.29308754529825093,"brier":0.08349176183235367,"accuracy":0.9104713952131475,"coverage":1},"0.4":{"converged":true,"loss":0.38116382634141843,"brier":0.11077322548983205,"accuracy":0.9098167375532361,"coverage":1}}},{"kind":"family","omitted":"omniscience","metrics":{"0.05":{"converged":true,"loss":0.402256944863963,"brier":0.12646860020668554,"accuracy":0.8223146790837447,"coverage":1},"0.1":{"converged":true,"loss":0.38732522750964793,"brier":0.12414235391576162,"accuracy":0.8220423385359026,"coverage":1},"0.2":{"converged":true,"loss":0.3897754688833965,"brier":0.1245518875248039,"accuracy":0.8222004253974464,"coverage":1},"0.4":{"converged":true,"loss":0.4369215321270542,"brier":0.13733969041886793,"accuracy":0.8221838981346494,"coverage":1}}},{"kind":"family","omitted":"scicode","metrics":{"0.05":{"converged":true,"loss":0.25473275543651214,"brier":0.07767662096937665,"accuracy":0.8896257750221324,"coverage":1},"0.1":{"converged":true,"loss":0.27131608900778154,"brier":0.081100964478908,"accuracy":0.8880775760259682,"coverage":1},"0.2":{"converged":true,"loss":0.31196704878414233,"brier":0.09106998273958045,"accuracy":0.8864426483613664,"coverage":1},"0.4":{"converged":true,"loss":0.39906266297472015,"brier":0.11796118993758795,"accuracy":0.883442205491569,"coverage":1}}},{"kind":"family","omitted":"tau-bench","metrics":{"0.05":{"converged":true,"loss":0.43286215597805777,"brier":0.13384562153769394,"accuracy":0.8096307937436563,"coverage":1},"0.1":{"converged":true,"loss":0.4194665641488263,"brier":0.13228354569775486,"accuracy":0.809379368491813,"coverage":1},"0.2":{"converged":true,"loss":0.42274521237159124,"brier":0.13381670067298987,"accuracy":0.8091874810084501,"coverage":1},"0.4":{"converged":true,"loss":0.4675060822170624,"brier":0.14759675425629296,"accuracy":0.8087073942396522,"coverage":1}}},{"kind":"family","omitted":"terminal-bench","metrics":{"0.05":{"converged":true,"loss":0.31477088671715564,"brier":0.08716921566344404,"accuracy":0.8616545961029392,"coverage":0.9937575148076612},"0.1":{"converged":true,"loss":0.3276888807536204,"brier":0.09026738657961986,"accuracy":0.8610286833958364,"coverage":0.9937575148076612},"0.2":{"converged":true,"loss":0.3610116898860214,"brier":0.09910114582011925,"accuracy":0.8587739278418092,"coverage":0.9937575148076612},"0.4":{"converged":true,"loss":0.43510751708730216,"brier":0.12298054774517282,"accuracy":0.8616994407437584,"coverage":0.9937575148076612}}},{"kind":"family","omitted":"arc-agi","metrics":{"0.05":{"converged":true,"loss":0.37376402486529803,"brier":0.10814168205344081,"accuracy":0.8538223611382457,"coverage":0.5819690768531879},"0.1":{"converged":true,"loss":0.3663696110523547,"brier":0.10960578495202154,"accuracy":0.8515107117275535,"coverage":0.5819690768531879},"0.2":{"converged":true,"loss":0.38167655268425377,"brier":0.11643544154527691,"accuracy":0.8484306830761243,"coverage":0.5819690768531879},"0.4":{"converged":true,"loss":0.4396906381320429,"brier":0.13652271673675898,"accuracy":0.8411441036660829,"coverage":0.5819690768531879}}},{"kind":"family","omitted":"deepswe","metrics":{"0.05":{"converged":true,"loss":0.4135311483245137,"brier":0.13237745105842336,"accuracy":0.7964929214929215,"coverage":1},"0.1":{"converged":true,"loss":0.4295338525426164,"brier":0.13601820603787804,"accuracy":0.8023487773487776,"coverage":1},"0.2":{"converged":true,"loss":0.46435624415969096,"brier":0.1463920797616732,"accuracy":0.8033140283140288,"coverage":1},"0.4":{"converged":true,"loss":0.5282346291092486,"brier":0.17050068407003197,"accuracy":0.8019948519948522,"coverage":1}}},{"kind":"family","omitted":"swe-bench-pro","metrics":{"0.05":{"converged":true,"loss":null,"brier":null,"accuracy":null,"coverage":0},"0.1":{"converged":true,"loss":null,"brier":null,"accuracy":null,"coverage":0},"0.2":{"converged":true,"loss":null,"brier":null,"accuracy":null,"coverage":0},"0.4":{"converged":true,"loss":null,"brier":null,"accuracy":null,"coverage":0}}},{"kind":"family","omitted":"arc-agi-3","metrics":{"0.05":{"converged":true,"loss":0.2411093059351158,"brier":0.06983104237068179,"accuracy":0.9083333333333333,"coverage":1},"0.1":{"converged":true,"loss":0.26787650406742386,"brier":0.07602109846209174,"accuracy":0.9083333333333333,"coverage":1},"0.2":{"converged":true,"loss":0.32117343084208333,"brier":0.09054268865400376,"accuracy":0.9083333333333333,"coverage":1},"0.4":{"converged":true,"loss":0.4198502890177224,"brier":0.12412983307869115,"accuracy":0.9083333333333333,"coverage":1}}},{"kind":"family","omitted":"frontiercode","metrics":{"0.05":{"converged":true,"loss":0.30972081961270215,"brier":0.09567090699357882,"accuracy":0.8643526350032384,"coverage":0.97562523803478},"0.1":{"converged":true,"loss":0.32880490862633477,"brier":0.09986030386582034,"accuracy":0.8601366297982938,"coverage":0.97562523803478},"0.2":{"converged":true,"loss":0.37103741402838786,"brier":0.1114095063800974,"accuracy":0.8552700065061666,"coverage":0.97562523803478},"0.4":{"converged":true,"loss":0.45323986842472025,"brier":0.1396310262959477,"accuracy":0.8537518976360734,"coverage":0.97562523803478}}},{"kind":"family","omitted":"enigma-eval","metrics":{"0.05":{"converged":true,"loss":0.5051755525295909,"brier":0.17640986755435334,"accuracy":0.7,"coverage":1},"0.1":{"converged":true,"loss":0.508509619847346,"brier":0.17340028465107704,"accuracy":0.7,"coverage":1},"0.2":{"converged":true,"loss":0.5274610964705959,"brier":0.17629345468769092,"accuracy":0.7,"coverage":1},"0.4":{"converged":true,"loss":0.5716919286646975,"brier":0.1922139656321839,"accuracy":0.7,"coverage":1}}},{"kind":"family","omitted":"swe-atlas-qna","metrics":{"0.05":{"converged":true,"loss":0.5562798252449502,"brier":0.17784228730432883,"accuracy":0.8333333333333334,"coverage":1},"0.1":{"converged":true,"loss":0.5413961243864148,"brier":0.17643262919533587,"accuracy":0.8333333333333334,"coverage":1},"0.2":{"converged":true,"loss":0.5466515417586926,"brier":0.18052466499410738,"accuracy":0.8333333333333334,"coverage":1},"0.4":{"converged":true,"loss":0.5820143718087982,"brier":0.19615659447788444,"accuracy":0.8333333333333334,"coverage":1}}},{"kind":"family","omitted":"swe-atlas-refactoring","metrics":{"0.05":{"converged":true,"loss":0.28427808491291423,"brier":0.0818272768480202,"accuracy":1,"coverage":1},"0.1":{"converged":true,"loss":0.33645703952267353,"brier":0.09997953764476233,"accuracy":1,"coverage":1},"0.2":{"converged":true,"loss":0.4091885292433835,"brier":0.12503218830645302,"accuracy":1,"coverage":1},"0.4":{"converged":true,"loss":0.5056462008078123,"brier":0.16185648128375096,"accuracy":0.6,"coverage":1}}},{"kind":"family","omitted":"swe-atlas-test-writing","metrics":{"0.05":{"converged":true,"loss":0.40933724819827183,"brier":0.12865324510596032,"accuracy":0.9166666666666666,"coverage":1},"0.1":{"converged":true,"loss":0.42837903655644133,"brier":0.13360074591417173,"accuracy":0.9166666666666666,"coverage":1},"0.2":{"converged":true,"loss":0.4678902945678929,"brier":0.14667575014760723,"accuracy":0.9166666666666666,"coverage":1},"0.4":{"converged":true,"loss":0.5374954062978582,"brier":0.17509923675991054,"accuracy":0.9166666666666666,"coverage":1}}},{"kind":"family","omitted":"livebench-math","metrics":{"0.05":{"converged":true,"loss":0.5849385337674865,"brier":0.16361656844500677,"accuracy":0.7101139601139564,"coverage":1},"0.1":{"converged":true,"loss":0.5643939282486088,"brier":0.15955348285996215,"accuracy":0.7079772079772043,"coverage":1},"0.2":{"converged":true,"loss":0.5533506384410798,"brier":0.15692474277220583,"accuracy":0.708689458689455,"coverage":1},"0.4":{"converged":true,"loss":0.5694183331498113,"brier":0.1625055025491982,"accuracy":0.7094017094017058,"coverage":1}}},{"kind":"family","omitted":"livebench-coding","metrics":{"0.05":{"converged":true,"loss":0.6626930449499328,"brier":0.19313372588987646,"accuracy":0.6809116809116733,"coverage":1},"0.1":{"converged":true,"loss":0.6329921049656423,"brier":0.18768201196833817,"accuracy":0.6780626780626704,"coverage":1},"0.2":{"converged":true,"loss":0.6084261286769279,"brier":0.1821576898933469,"accuracy":0.673789173789166,"coverage":1},"0.4":{"converged":true,"loss":0.6056449317041368,"brier":0.18182312167702516,"accuracy":0.6737891737891659,"coverage":1}}},{"kind":"family","omitted":"livebench-language","metrics":{"0.05":{"converged":true,"loss":0.5546245043453517,"brier":0.16456633994013886,"accuracy":0.7141500474833817,"coverage":1},"0.1":{"converged":true,"loss":0.5401040389741242,"brier":0.16147497375362052,"accuracy":0.7141500474833817,"coverage":1},"0.2":{"converged":true,"loss":0.5357050519353466,"brier":0.16031213897143778,"accuracy":0.7103513770180445,"coverage":1},"0.4":{"converged":true,"loss":0.5586036828787035,"brier":0.16804352058471397,"accuracy":0.7094017094017103,"coverage":1}}},{"kind":"family","omitted":"livebench-data","metrics":{"0.05":{"converged":true,"loss":0.9111069240729588,"brier":0.268116976165901,"accuracy":0.5902184235517569,"coverage":1},"0.1":{"converged":true,"loss":0.8349556930542145,"brier":0.2566102870671265,"accuracy":0.5883190883190884,"coverage":1},"0.2":{"converged":true,"loss":0.7550449001936124,"brier":0.2407050770295823,"accuracy":0.5883190883190884,"coverage":1},"0.4":{"converged":true,"loss":0.6927827294910385,"brier":0.22340916191281077,"accuracy":0.5873694207027541,"coverage":1}}},{"kind":"family","omitted":"multi-swe-bench","metrics":{"0.05":{"converged":true,"loss":0.5837544187172861,"brier":0.1535258256740078,"accuracy":0.7198480531813876,"coverage":1},"0.1":{"converged":true,"loss":0.5624149897856695,"brier":0.14902325046860232,"accuracy":0.7169990503323846,"coverage":1},"0.2":{"converged":true,"loss":0.5505415043058572,"brier":0.1456955112676427,"accuracy":0.7169990503323846,"coverage":1},"0.4":{"converged":true,"loss":0.5663741379650836,"brier":0.1507470405150744,"accuracy":0.717948717948719,"coverage":1}}},{"kind":"family","omitted":"livebench-reasoning","metrics":{"0.05":{"converged":true,"loss":0.5840343157898359,"brier":0.15702272277829288,"accuracy":0.7211538461538425,"coverage":1},"0.1":{"converged":true,"loss":0.5640912614711239,"brier":0.15325212832937016,"accuracy":0.7190170940170904,"coverage":1},"0.2":{"converged":true,"loss":0.5532292940679057,"brier":0.15088819815424018,"accuracy":0.7204415954415918,"coverage":1},"0.4":{"converged":true,"loss":0.5690278455705702,"brier":0.1566148873137895,"accuracy":0.7183048433048397,"coverage":1}}},{"kind":"family","omitted":"livebench-instructions","metrics":{"0.05":{"converged":true,"loss":0.6905213084247831,"brier":0.22958676749025828,"accuracy":0.658475783475779,"coverage":1},"0.1":{"converged":true,"loss":0.6508153677689584,"brier":0.22172459265080668,"accuracy":0.655626780626776,"coverage":1},"0.2":{"converged":true,"loss":0.6162596938338335,"brier":0.21292758762062053,"accuracy":0.655626780626776,"coverage":1},"0.4":{"converged":true,"loss":0.6066611845184384,"brier":0.2094933347420913,"accuracy":0.6570512820512775,"coverage":1}}},{"kind":"family","omitted":"swe-rebench","metrics":{"0.05":{"converged":true,"loss":0.4315407721502176,"brier":0.13697349724801527,"accuracy":0.8139861927895675,"coverage":1},"0.1":{"converged":true,"loss":0.4507267626303278,"brier":0.14355508694548774,"accuracy":0.8032472513423675,"coverage":1},"0.2":{"converged":true,"loss":0.4863432142202282,"brier":0.15602188105044085,"accuracy":0.8032472513423675,"coverage":1},"0.4":{"converged":true,"loss":0.5445209594015045,"brier":0.17869059162445594,"accuracy":0.7844541038097672,"coverage":1}}},{"kind":"family","omitted":"epoch-game-puzzles","metrics":{"0.05":{"converged":true,"loss":0.5060967550733103,"brier":0.14231589842068645,"accuracy":0.7757522605031528,"coverage":0.8539379457352089},"0.1":{"converged":true,"loss":0.46507287562409355,"brier":0.13646836838392307,"accuracy":0.7765807341052655,"coverage":0.8539379457352089},"0.2":{"converged":true,"loss":0.4413362290778648,"brier":0.13138399077227622,"accuracy":0.7749755665011718,"coverage":0.8539379457352089},"0.4":{"converged":true,"loss":0.4652031889736362,"brier":0.13674602136567923,"accuracy":0.7734933754473924,"coverage":0.8539379457352089}}},{"kind":"family","omitted":"frontiermath","metrics":{"0.05":{"converged":true,"loss":0.3569492715392873,"brier":0.11052665812736467,"accuracy":0.8353054974365407,"coverage":1},"0.1":{"converged":true,"loss":0.37350742611742466,"brier":0.11461858199866011,"accuracy":0.8369266183258413,"coverage":1},"0.2":{"converged":true,"loss":0.4097300536113381,"brier":0.12496137154340033,"accuracy":0.8331952452444339,"coverage":1},"0.4":{"converged":true,"loss":0.48076586800197924,"brier":0.1495968107714975,"accuracy":0.8335296513392095,"coverage":1}}},{"kind":"family","omitted":"simpleqa","metrics":{"0.05":{"converged":true,"loss":0.5451962296847168,"brier":0.18299263552616035,"accuracy":0.7291414752116109,"coverage":1},"0.1":{"converged":true,"loss":0.5320231063353241,"brier":0.1795730189050564,"accuracy":0.7267230955260002,"coverage":1},"0.2":{"converged":true,"loss":0.5327508985812097,"brier":0.17933258821459977,"accuracy":0.7190648931882332,"coverage":1},"0.4":{"converged":true,"loss":0.5613113982064745,"brier":0.18863504101529047,"accuracy":0.7093913744457905,"coverage":1}}},{"kind":"family","omitted":"vals-finance-agent","metrics":{"0.05":{"converged":true,"loss":0.5201231727194205,"brier":0.1757120299256055,"accuracy":0.741935483870968,"coverage":1},"0.1":{"converged":true,"loss":0.5081003406270211,"brier":0.17212057566896014,"accuracy":0.7440860215053765,"coverage":1},"0.2":{"converged":true,"loss":0.5068749958427969,"brier":0.17034205227784835,"accuracy":0.7483870967741937,"coverage":1},"0.4":{"converged":true,"loss":0.5351907809452034,"brier":0.17813923867550305,"accuracy":0.7505376344086023,"coverage":1}}},{"kind":"family","omitted":"vals-code-migration","metrics":{"0.05":{"converged":true,"loss":0.4786366640171628,"brier":0.16019202917213266,"accuracy":0.7563025210083995,"coverage":1},"0.1":{"converged":true,"loss":0.476010070756607,"brier":0.15893457966543129,"accuracy":0.7546218487394919,"coverage":1},"0.2":{"converged":true,"loss":0.4865446921515938,"brier":0.16088995626187383,"accuracy":0.7529411764705843,"coverage":1},"0.4":{"converged":true,"loss":0.5266042764822447,"brier":0.17361773848906975,"accuracy":0.7529411764705843,"coverage":1}}},{"kind":"family","omitted":"vals-legal-research","metrics":{"0.05":{"converged":true,"loss":0.401161910726958,"brier":0.12755217389188328,"accuracy":0.8068181818181818,"coverage":1},"0.1":{"converged":true,"loss":0.40941150372468554,"brier":0.12900083796791054,"accuracy":0.8068181818181818,"coverage":1},"0.2":{"converged":true,"loss":0.43374691645766167,"brier":0.135118675505833,"accuracy":0.8106060606060606,"coverage":1},"0.4":{"converged":true,"loss":0.49109361150859876,"brier":0.1542357686358119,"accuracy":0.8125,"coverage":1}}},{"kind":"family","omitted":"vals-excel-modeling","metrics":{"0.05":{"converged":true,"loss":0.39061263372727995,"brier":0.12718563295110083,"accuracy":0.8125,"coverage":1},"0.1":{"converged":true,"loss":0.40164734516706196,"brier":0.1293492952062106,"accuracy":0.8106060606060606,"coverage":1},"0.2":{"converged":true,"loss":0.4295545205840927,"brier":0.1364975802174529,"accuracy":0.8087121212121212,"coverage":1},"0.4":{"converged":true,"loss":0.4901828253541279,"brier":0.156759414517755,"accuracy":0.8087121212121212,"coverage":1}}},{"kind":"family","omitted":"vibe-code","metrics":{"0.05":{"converged":true,"loss":0.4097797659742801,"brier":0.1338604236952467,"accuracy":0.8087121212121212,"coverage":1},"0.1":{"converged":true,"loss":0.41781704189193447,"brier":0.13535588508795532,"accuracy":0.8068181818181818,"coverage":1},"0.2":{"converged":true,"loss":0.44209053824958433,"brier":0.14151174070599187,"accuracy":0.8068181818181818,"coverage":1},"0.4":{"converged":true,"loss":0.49860406601597884,"brier":0.16044374167477665,"accuracy":0.803030303030303,"coverage":1}}}]}