{"generatedAt":"2026-09-07T22:04:24.664Z","candidate":"cohort-source-1","scope":"Public external snapshots not used in candidate design, first acquired after this freeze. Earlier publication is allowed; this is not temporal validation. Include every exactly mapped configuration on reviewed comparable tasks, no selection by favorable outcomes. Freeze before fetching either source. Do not retune the candidate after seeing the external results.","inputHash":"sha256-225089c1849a37101b10a229a834dd94f12807a8dfe88240540ebb2abdfa411b","externalHash":"sha256-d5c8955b42711d483570ec27af0eeaa140388b8c6f9ae8df928c17398670b1cb","reviewHash":"sha256-03db48789645673ad7dc253ab2ae7b63af5e7b1c160bf323931ecf8752b193f5","implementationHash":"sha256-458670246765ecbf57dc3c12983e5a3d9b734998cd68ae868d31328c3073f0fc","passed":true,"failures":[],"coverage":0.9351851851851806,"predictablePairs":7596,"targetPairs":8194,"summary":{"equal-family":{"loss":0.6159173572940432,"brier":0.18659265703565536,"accuracy":0.6839060279654052},"model-source":{"loss":0.6122397534979619,"brier":0.1842389974171547,"accuracy":0.6931579146925393}},"byFamily":{"livebench-math":{"equal-family":{"loss":0.5753896948828103,"brier":0.16399870879793024,"accuracy":0.7153846153846234},"model-source":{"loss":0.565908894665568,"brier":0.1603647938760399,"accuracy":0.7123076923077003}},"livebench-coding":{"equal-family":{"loss":0.6803194523631692,"brier":0.20774650990524265,"accuracy":0.6399999999999944},"model-source":{"loss":0.6355532737069292,"brier":0.18790320048316642,"accuracy":0.6815384615384562}},"livebench-language":{"equal-family":{"loss":0.5619775445089404,"brier":0.17010794496881276,"accuracy":0.7164102564102547},"model-source":{"loss":0.5503900320412147,"brier":0.16546354938097907,"accuracy":0.7123076923076905}},"livebench-data":{"equal-family":{"loss":0.8412557290617196,"brier":0.2660652474993106,"accuracy":0.555384615384612},"model-source":{"loss":0.8293565181008591,"brier":0.25860110377594786,"accuracy":0.5779487179487148}},"multi-swe-bench":{"equal-family":{"loss":0.542334873989304,"brier":0.13965780829019583,"accuracy":0.727692307692306},"model-source":{"loss":0.5463700905364829,"brier":0.14099107850197598,"accuracy":0.7338461538461523}},"livebench-reasoning":{"equal-family":{"loss":0.611050687197801,"brier":0.1710612437417532,"accuracy":0.6888461538461638},"model-source":{"loss":0.5800020479721829,"brier":0.15797382450536127,"accuracy":0.7119230769230864}},"livebench-instructions":{"equal-family":{"loss":0.5803727037321246,"brier":0.19677821855104385,"accuracy":0.7073076923077023},"model-source":{"loss":0.6616418493526296,"brier":0.22683408153858328,"accuracy":0.6550000000000118}},"swe-rebench":{"equal-family":{"loss":0.5406588529628559,"brier":0.1780120250868395,"accuracy":0.7175324675324677},"model-source":{"loss":0.5348837980440524,"brier":0.17640691395234057,"accuracy":0.7554112554112553}}},"familyResampling":{"draws":10000,"lossDifferenceRange":[-0.024298707264990227,0.023341224100103822],"scope":"Exploratory central 95% range from resampling whole families. Families within an evaluator remain correlated; this is not proof of statistical superiority. Negative favors the candidate."},"bySource":{"LiveBench":{"equal-family":{"loss":0.6275286693908539,"brier":0.18791652596490113,"accuracy":0.678717948717915},"model-source":{"loss":0.6241746723394218,"brier":0.18544737600886885,"accuracy":0.6835531135530796}},"SWE-rebench":{"equal-family":{"loss":0.5406588529628559,"brier":0.1780120250868395,"accuracy":0.7175324675324677},"model-source":{"loss":0.5348837980440524,"brier":0.17640691395234057,"accuracy":0.7554112554112553}}},"sources":{"livebench":{"rawModels":56,"mappedModels":27,"skipped":[{"model":"deepseek-v4-flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"deepseek-v4-pro","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"glm-5.2","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-5.2-codex","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"grok-4.3","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"grok-4.5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"grok-build-0.1","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"kimi-k2.6-thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"kimi-k2.7-code","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"minimax-m3","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"qwen3.6-27b","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"qwen3.6-plus","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"qwen3.7-max","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"kimi-k3","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"inkling-xhigh","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"deepseek-v4-flash-0731","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"qwen3.8-max","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"smaug-agentic","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"smaug-mini","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"deepseek-v4-pro-0813","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"grok-4.6","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"qwen3.8-27b","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"ox-alpha-max","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"deepseek-v4-flash-vision-exp","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"glm-5.3","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"glm-5.3-flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"qwen3.8-flash-next","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"nemotron-3-ultra-550b-a55b","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"smaug-flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"}]},"sweRebench":{"rawModels":117,"mappedModels":23,"skipped":[{"model":"gpt-4.1-2025-04-14","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"DeepSeek-V3-0324","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"DeepSeek-V3","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-235B-A22B no-thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-235B-A22B thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-32B no-thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-32B thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Llama-4-Maverick-17B-128E-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen2.5-72B-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Llama-3.3-70B-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Llama-4-Scout-17B-16E-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gemma-3-27b-it","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen2.5-Coder-32B-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-4.1-mini-2025-04-14","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-4.1-nano-2025-04-14","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gemini-2.0-flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gemini-2.5-flash-preview-05-20 no-thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Devstral-Small-2505","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"DeepSeek-V3-0324","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Claude Sonnet 3.5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Claude Sonnet 4","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gemini-2.5-flash-preview-05-20 no-thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-4.1-2025-04-14","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"o3-2025-04-16","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gemini-2.5-flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gemini-2.0-flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gemini-2.5-pro","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-Coder-480B-A35B-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-235B-A22B-Thinking-2507","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-235B-A22B-Instruct-2507","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"o4-mini-2025-04-16","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"DeepSeek-R1-0528","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"horizon-alpha","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-Coder-30B-A3B-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"horizon-beta","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-4.5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-4.5 Air","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Grok Code Fast 1","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-oss-120b","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-oss-20b","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-4.1-mini-2025-04-14","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Kimi K2","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-235B-A22B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-32B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Kimi K2 Instruct 0905","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-Next-80B-A3B-Instruct","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"DeepSeek-V3.1","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Grok 4","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Claude Sonnet 4.5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Claude Opus 4.1","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-5-codex","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-30B-A3B-Thinking-2507","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-30B-A3B-Instruct-2507","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-4.6","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"MiniMax M2","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Claude Opus 4.5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Gemini 3 Pro Preview","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-5.1-codex-max","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Devstral-2-123B-Instruct-2512","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Devstral-Small-2-24B-Instruct-2512","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"DeepSeek-V3.2","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Claude Code","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"MiniMax M2.1","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-5.1-codex","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Kimi K2 Thinking","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-4.7","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Gemini 3 Flash Preview","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-4.7 Flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Kimi K2.5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-5.2-codex","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3-Coder-Next","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Codex","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"MiniMax M2.5","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Junie","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Gemini 3.1 Pro Preview","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Claude Sonnet 4.6","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"gpt-5.3-codex","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3.5-35B-A3B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3.5-397B-A17B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Step-3.5-Flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-5.1","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"MiniMax M2.7","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3.5-27B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Gemma 4 31B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Cursor","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"GLM-5.1","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Kimi K2.6","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Gemini 3.5 Flash","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"MiniMax M3","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"MiMo V2.5 Pro","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3.6-27B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"Qwen3.6-35B-A3B","reason":"Model checkpoint or effort has no explicit reviewed mapping"},{"model":"DeepSeek-V4 Flash [high]","reason":"Model checkpoint or effort has no explicit reviewed mapping"}]}}}