{"schemaVersion":2,"generatedAt":"2026-11-03T06:35:53.431491+00:00","method":"Bayesian two-parameter logistic capability model on benchmark score levels","capabilityUnit":"model_reasoning_state","intervalType":"95% posterior credible intervals from MCLMC","battery":{"models":889,"rankedModels":886,"reasoningStates":1917,"capabilityUnit":"model_reasoning_state","benchmarks":750,"filledPct":2.7},"rankingMinBenchmarks":4,"signal":{"sharedVariancePct":62.4,"uniqueVariancePct":37.6},"fitMetrics":{"rhat_max":1.0175749137970191,"ess_min":774.708526271153,"per_variable":{"phi_state":{"ess_min":809.454212192043,"rhat_max":1.0175749137970191},"beta":{"ess_min":787.3079213601404,"rhat_max":1.0121124116752545},"difficulty":{"ess_min":774.708526271153,"rhat_max":1.0154845179317882}},"diagnostics_engine":"arviz","divergences":0,"sampler":{"sampler":"mclmc","chains":8,"warmup":300,"samples":450,"step_size":0.175,"seed":123},"observations_sha256":"6d913591ac5618464ec4a2b5f002a209459a2e7341ef05944934bce7f4dc78f6","packages":{"nutpie":"0.16.11","pymc":"6.3.2","pytensor":"3.3.4","jax":"0.12.0.1","jaxlib":"0.12.0.1","numpy":"2.5.4"}},"difficulty":{"definition":"position on the benchmark's axis (base capability plus the benchmark's reasoning scale times the reasoning offset; the capability score itself for a default state) at which the expected score reaches half of the benchmark's range above its guessing floor, from the fitted logistic item curve","scale":"display","reliableBenchmarkCount":750,"aime2025":{"difficulty":131.6,"extrapolated":false},"saturationDefinition":"when the frontier (the best rank-eligible state released by each date) first reaches the benchmark's 90% point on its own axis; not yet reached, projected along the frontier's linear trend over its record-setting states in the last 2 years, per posterior draw","saturationInterval":80.0,"frontierAsOf":"2026-11-01","frontierPointsPerYear":[15.2,13.6,17.0],"slugs":[{"slug":"medic-summarization","family":"medic-summarization","name":"MEDIC (clinical summarization)","nStates":49,"nModels":49,"difficulty":5.0,"saturation":157.2,"saturationExtrapolated":false,"frontierPct":91.6,"extrapolated":true,"extrapolation":"below","maxTested":147.0,"saturationDate":"2025-12-11","saturationDateLo":"2025-06-09","saturationDateHi":"2026-04-07","saturated":true},{"slug":"pope-f1","family":"pope-f1","name":"POPE (F1, pooled over three settings)","nStates":16,"nModels":15,"difficulty":22.4,"saturation":135.5,"saturationExtrapolated":false,"frontierPct":95.0,"extrapolated":true,"extrapolation":"below","maxTested":150.1,"saturationDate":"2024-09-12","saturationDateLo":"2024-05-13","saturationDateHi":"2025-03-24","saturated":true},{"slug":"lambada","family":"lambada","name":"LAMBADA","nStates":34,"nModels":34,"difficulty":32.7,"saturation":150.4,"saturationExtrapolated":false,"frontierPct":92.7,"extrapolated":false,"extrapolation":null,"maxTested":116.1,"saturationDate":"2025-07-31","saturationDateLo":"2025-01-30","saturationDateHi":"2026-02-04","saturated":true},{"slug":"ocrbench","family":"ocrbench","name":"OCRBench","nStates":92,"nModels":73,"difficulty":37.3,"saturation":164.8,"saturationExtrapolated":false,"frontierPct":91.3,"extrapolated":true,"extrapolation":"below","maxTested":161.4,"saturationDate":"2026-02-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-09-02","saturated":true},{"slug":"docvqa","family":"docvqa","name":"DocVQA","nStates":66,"nModels":58,"difficulty":43.8,"saturation":122.5,"saturationExtrapolated":false,"frontierPct":97.5,"extrapolated":true,"extrapolation":"below","maxTested":152.3,"saturationDate":"2023-11-05","saturationDateLo":"2023-03-13","saturationDateHi":"2024-04-09","saturated":true},{"slug":"textvqa-val","family":"textvqa-val","name":"TextVQA (validation)","nStates":23,"nModels":21,"difficulty":51.7,"saturation":151.4,"saturationExtrapolated":false,"frontierPct":93.1,"extrapolated":true,"extrapolation":"below","maxTested":152.0,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-12-11","saturated":true},{"slug":"c3","family":"c3","name":"C3","nStates":11,"nModels":11,"difficulty":57.6,"saturation":164.8,"saturationExtrapolated":false,"frontierPct":91.3,"extrapolated":true,"extrapolation":"below","maxTested":123.9,"saturationDate":"2026-04-07","saturationDateLo":"2025-07-31","saturationDateHi":"2027-05-15","saturated":true},{"slug":"docvqa-val","family":"docvqa-val","name":"DocVQA (val)","nStates":9,"nModels":8,"difficulty":62.9,"saturation":114.0,"saturationExtrapolated":false,"frontierPct":99.2,"extrapolated":true,"extrapolation":"below","maxTested":149.8,"saturationDate":"2023-03-13","saturationDateLo":"2023-03-13","saturationDateHi":"2023-06-13","saturated":true},{"slug":"chartqa-test","family":"chartqa-test","name":"ChartQA (test)","nStates":73,"nModels":65,"difficulty":63.4,"saturation":159.0,"saturationExtrapolated":false,"frontierPct":93.3,"extrapolated":true,"extrapolation":"below","maxTested":153.0,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2026-02-05","saturated":true},{"slug":"chid","family":"chid","name":"ChID","nStates":8,"nModels":8,"difficulty":63.6,"saturation":134.3,"saturationExtrapolated":false,"frontierPct":97.3,"extrapolated":true,"extrapolation":"below","maxTested":115.1,"saturationDate":"2024-08-29","saturationDateLo":"2023-03-13","saturationDateHi":"2025-12-11","saturated":true},{"slug":"superglue","family":"superglue","name":"SuperGLUE","nStates":8,"nModels":8,"difficulty":64.4,"saturation":300.7,"saturationExtrapolated":true,"frontierPct":73.7,"extrapolated":false,"extrapolation":null,"maxTested":88.1,"saturationDate":"2034-04-18","saturationDateLo":"2030-08-24","saturationDateHi":"2040-06-11","saturated":false},{"slug":"seed-bench-image","family":"seed-bench-image","name":"SEED-Bench (image)","nStates":16,"nModels":15,"difficulty":65.5,"saturation":232.1,"saturationExtrapolated":true,"frontierPct":85.2,"extrapolated":true,"extrapolation":"below","maxTested":149.2,"saturationDate":"2030-09-18","saturationDateLo":"2029-05-01","saturationDateHi":"2032-04-01","saturated":false},{"slug":"ifeval-helm-capabilities","family":"ifeval-helm-capabilities","name":"IFEval (HELM Capabilities)","nStates":66,"nModels":64,"difficulty":65.6,"saturation":154.4,"saturationExtrapolated":false,"frontierPct":92.8,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2025-10-06","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-11","saturated":true},{"slug":"piqa","family":"piqa","name":"PIQA","nStates":81,"nModels":81,"difficulty":66.2,"saturation":209.6,"saturationExtrapolated":true,"frontierPct":91.4,"extrapolated":false,"extrapolation":null,"maxTested":129.3,"saturationDate":"2029-05-14","saturationDateLo":"2028-09-25","saturationDateHi":"2030-01-02","saturated":false},{"slug":"squad2","family":"squad2","name":"SQuAD 2.0","nStates":27,"nModels":27,"difficulty":66.4,"saturation":130.7,"saturationExtrapolated":false,"frontierPct":97.2,"extrapolated":false,"extrapolation":null,"maxTested":116.9,"saturationDate":"2024-06-19","saturationDateLo":"2023-11-05","saturationDateHi":"2024-12-04","saturated":true},{"slug":"hellaswag-openllm-v1-harness","family":"hellaswag-openllm-v1-harness","name":"HellaSwag (Open LLM Leaderboard v1)","nStates":118,"nModels":118,"difficulty":70.3,"saturation":131.6,"saturationExtrapolated":false,"frontierPct":97.9,"extrapolated":false,"extrapolation":null,"maxTested":121.7,"saturationDate":"2024-08-29","saturationDateLo":"2024-05-13","saturationDateHi":"2024-09-12","saturated":true},{"slug":"mmbench-en-test","family":"mmbench-en-test","name":"MMBench (English, test)","nStates":8,"nModels":8,"difficulty":72.7,"saturation":137.7,"saturationExtrapolated":false,"frontierPct":96.7,"extrapolated":true,"extrapolation":"below","maxTested":131.0,"saturationDate":"2024-09-12","saturationDateLo":"2024-08-29","saturationDateHi":"2025-03-24","saturated":true},{"slug":"wildbench-helm-capabilities","family":"wildbench-helm-capabilities","name":"WildBench v2 (HELM Capabilities)","nStates":57,"nModels":55,"difficulty":73.1,"saturation":164.0,"saturationExtrapolated":false,"frontierPct":90.7,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2026-06-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-01","saturated":true},{"slug":"hellaswag-unspecified-scoring-protocol","family":"hellaswag-unspecified-scoring-protocol","name":"HellaSwag (Unspecified Scoring Protocol)","nStates":118,"nModels":118,"difficulty":74.6,"saturation":138.0,"saturationExtrapolated":false,"frontierPct":97.3,"extrapolated":false,"extrapolation":null,"maxTested":132.0,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"countbenchqa","family":"countbenchqa","name":"CountBenchQA","nStates":11,"nModels":10,"difficulty":75.1,"saturation":134.4,"saturationExtrapolated":false,"frontierPct":97.2,"extrapolated":true,"extrapolation":"below","maxTested":149.5,"saturationDate":"2024-09-12","saturationDateLo":"2024-05-13","saturationDateHi":"2024-12-04","saturated":true},{"slug":"browsecomp-long-context-128k","family":"browsecomp-long-context-128k","name":"BrowseComp Long Context - 128K","nStates":10,"nModels":10,"difficulty":75.2,"saturation":152.3,"saturationExtrapolated":false,"frontierPct":94.3,"extrapolated":true,"extrapolation":"below","maxTested":155.3,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2026-02-11","saturated":true},{"slug":"mt-bench","family":"mt-bench","name":"MT-Bench","nStates":41,"nModels":41,"difficulty":76.2,"saturation":125.8,"saturationExtrapolated":false,"frontierPct":98.4,"extrapolated":true,"extrapolation":"below","maxTested":125.8,"saturationDate":"2024-04-09","saturationDateLo":"2023-11-05","saturationDateHi":"2024-05-13","saturated":true},{"slug":"ccpm","family":"ccpm","name":"CCPM","nStates":11,"nModels":11,"difficulty":76.8,"saturation":148.5,"saturationExtrapolated":false,"frontierPct":96.7,"extrapolated":true,"extrapolation":"below","maxTested":123.9,"saturationDate":"2025-03-24","saturationDateLo":"2024-09-12","saturationDateHi":"2026-11-10","saturated":true},{"slug":"trivia-qa","family":"trivia-qa","name":"TriviaQA","nStates":58,"nModels":58,"difficulty":76.8,"saturation":138.4,"saturationExtrapolated":false,"frontierPct":96.4,"extrapolated":false,"extrapolation":null,"maxTested":132.0,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"refcoco-8-split-average","family":"refcoco-8-split-average","name":"RefCOCO, RefCOCO+ and RefCOCOg eight-split average","nStates":9,"nModels":8,"difficulty":77.5,"saturation":156.9,"saturationExtrapolated":false,"frontierPct":93.8,"extrapolated":true,"extrapolation":"below","maxTested":151.2,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2026-11-07","saturated":true},{"slug":"instruction-following-scale-ai-2","family":"instruction-following-scale-ai-2","name":"Instruction Following","nStates":19,"nModels":19,"difficulty":77.6,"saturation":145.2,"saturationExtrapolated":false,"frontierPct":95.4,"extrapolated":true,"extrapolation":"below","maxTested":140.8,"saturationDate":"2025-03-24","saturationDateLo":"2024-12-04","saturationDateHi":"2025-07-31","saturated":true},{"slug":"mt-bench-gpt-4-turbo","family":"mt-bench-gpt-4-turbo","name":"MT-Bench (GPT-4-Turbo judge, corrected references)","nStates":18,"nModels":18,"difficulty":77.7,"saturation":140.6,"saturationExtrapolated":false,"frontierPct":96.3,"extrapolated":true,"extrapolation":"below","maxTested":130.0,"saturationDate":"2024-12-04","saturationDateLo":"2024-09-12","saturationDateHi":"2025-04-15","saturated":true},{"slug":"omnidocbench-v1-5","family":"omnidocbench-v1-5","name":"OmniDocBench v1.5","nStates":43,"nModels":38,"difficulty":79.2,"saturation":164.0,"saturationExtrapolated":false,"frontierPct":91.3,"extrapolated":true,"extrapolation":"below","maxTested":164.5,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-02","saturated":true},{"slug":"siqa-olmo3-mc","family":"siqa-olmo3-mc","name":"Social IQa (OLMo 3 5-shot multiple choice)","nStates":9,"nModels":9,"difficulty":79.9,"saturation":154.4,"saturationExtrapolated":false,"frontierPct":96.1,"extrapolated":true,"extrapolation":"below","maxTested":122.3,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2026-09-02","saturated":true},{"slug":"facts-grounding","family":"facts-grounding","name":"FACTS Grounding","nStates":23,"nModels":19,"difficulty":80.0,"saturation":168.3,"saturationExtrapolated":true,"frontierPct":91.3,"extrapolated":true,"extrapolation":"below","maxTested":147.1,"saturationDate":"2026-09-01","saturationDateLo":"2025-12-11","saturationDateHi":"2027-07-09","saturated":true},{"slug":"cmath","family":"cmath","name":"CMATH","nStates":29,"nModels":29,"difficulty":80.1,"saturation":135.8,"saturationExtrapolated":false,"frontierPct":97.3,"extrapolated":true,"extrapolation":"below","maxTested":132.0,"saturationDate":"2024-09-12","saturationDateLo":"2024-06-19","saturationDateHi":"2025-03-24","saturated":true},{"slug":"cmrc-2018","family":"cmrc-2018","name":"CMRC 2018","nStates":8,"nModels":8,"difficulty":80.7,"saturation":155.5,"saturationExtrapolated":false,"frontierPct":94.3,"extrapolated":true,"extrapolation":"below","maxTested":123.9,"saturationDate":"2025-07-31","saturationDateLo":"2024-12-17","saturationDateHi":"2027-01-13","saturated":true},{"slug":"ai2d-vlmevalkit","family":"ai2d-vlmevalkit","name":"AI2D (OpenVLM)","nStates":58,"nModels":58,"difficulty":80.9,"saturation":162.9,"saturationExtrapolated":false,"frontierPct":93.7,"extrapolated":false,"extrapolation":null,"maxTested":149.3,"saturationDate":"2026-02-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-10-28","saturated":true},{"slug":"cc-ocr","family":"cc-ocr","name":"CC-OCR","nStates":50,"nModels":41,"difficulty":81.4,"saturation":198.6,"saturationExtrapolated":true,"frontierPct":85.5,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2028-01-22","saturationDateLo":"2027-05-05","saturationDateHi":"2028-12-19","saturated":false},{"slug":"infovqa","family":"infovqa","name":"InfographicVQA","nStates":44,"nModels":37,"difficulty":82.1,"saturation":173.0,"saturationExtrapolated":false,"frontierPct":91.4,"extrapolated":false,"extrapolation":null,"maxTested":157.7,"saturationDate":"2026-02-04","saturationDateLo":"2025-07-31","saturationDateHi":"2026-10-26","saturated":true},{"slug":"vqav2-val","family":"vqav2-val","name":"VQAv2 (validation)","nStates":10,"nModels":10,"difficulty":82.5,"saturation":176.7,"saturationExtrapolated":true,"frontierPct":89.7,"extrapolated":true,"extrapolation":"below","maxTested":133.6,"saturationDate":"2026-12-26","saturationDateLo":"2025-12-11","saturationDateHi":"2028-11-14","saturated":false},{"slug":"drop","family":"drop","name":"DROP","nStates":70,"nModels":64,"difficulty":82.7,"saturation":149.0,"saturationExtrapolated":false,"frontierPct":94.5,"extrapolated":false,"extrapolation":null,"maxTested":148.5,"saturationDate":"2025-04-15","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"humaneval-mul","family":"humaneval-mul","name":"HumanEval-Mul","nStates":18,"nModels":18,"difficulty":84.0,"saturation":173.7,"saturationExtrapolated":true,"frontierPct":89.4,"extrapolated":false,"extrapolation":null,"maxTested":133.6,"saturationDate":"2026-11-17","saturationDateLo":"2025-12-11","saturationDateHi":"2028-01-16","saturated":false},{"slug":"realworldqa-reported","family":"realworldqa-reported","name":"RealWorldQA - release-reported visual perception","nStates":104,"nModels":91,"difficulty":85.1,"saturation":188.0,"saturationExtrapolated":true,"frontierPct":86.1,"extrapolated":false,"extrapolation":null,"maxTested":162.1,"saturationDate":"2027-10-27","saturationDateLo":"2027-06-05","saturationDateHi":"2028-04-07","saturated":false},{"slug":"openbookqa-helm-lite","family":"openbookqa-helm-lite","name":"OpenBookQA (HELM Lite, 5-shot multiple choice)","nStates":72,"nModels":72,"difficulty":85.2,"saturation":120.4,"saturationExtrapolated":false,"frontierPct":99.6,"extrapolated":false,"extrapolation":null,"maxTested":133.6,"saturationDate":"2023-06-13","saturationDateLo":"2023-03-13","saturationDateHi":"2023-11-05","saturated":true},{"slug":"legal_bench","family":"legal_bench","name":"LegalBench","nStates":145,"nModels":136,"difficulty":85.3,"saturation":179.4,"saturationExtrapolated":true,"frontierPct":87.2,"extrapolated":false,"extrapolation":null,"maxTested":166.5,"saturationDate":"2027-06-21","saturationDateLo":"2027-03-11","saturationDateHi":"2027-10-01","saturated":false},{"slug":"pubmedqa-openmedical-harness","family":"pubmedqa-openmedical-harness","name":"PubMedQA (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":85.6,"saturation":213.7,"saturationExtrapolated":true,"frontierPct":88.4,"extrapolated":false,"extrapolation":null,"maxTested":103.7,"saturationDate":"2029-02-03","saturationDateLo":"2026-12-07","saturationDateHi":"2032-06-11","saturated":false},{"slug":"countbench","family":"countbench","name":"CountBench","nStates":54,"nModels":44,"difficulty":85.8,"saturation":136.2,"saturationExtrapolated":false,"frontierPct":97.7,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2024-09-12","saturated":true},{"slug":"bool-q","family":"bool-q","name":"BoolQ","nStates":55,"nModels":55,"difficulty":86.0,"saturation":151.7,"saturationExtrapolated":false,"frontierPct":97.1,"extrapolated":false,"extrapolation":null,"maxTested":120.3,"saturationDate":"2025-07-31","saturationDateLo":"2024-12-15","saturationDateHi":"2026-04-07","saturated":true},{"slug":"mmbench-v1-1-dev-en","family":"mmbench-v1-1-dev-en","name":"MMBench v1.1 (dev, English)","nStates":56,"nModels":46,"difficulty":86.3,"saturation":146.7,"saturationExtrapolated":false,"frontierPct":95.9,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2025-03-24","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"mbpp-evalplus-base","family":"mbpp-evalplus-base","name":"MBPP EvalPlus (base tests)","nStates":10,"nModels":10,"difficulty":88.9,"saturation":131.5,"saturationExtrapolated":false,"frontierPct":98.5,"extrapolated":true,"extrapolation":"below","maxTested":130.0,"saturationDate":"2024-08-29","saturationDateLo":"2024-05-13","saturationDateHi":"2024-09-12","saturated":true},{"slug":"race-middle","family":"race-middle","name":"RACE-Middle","nStates":27,"nModels":27,"difficulty":89.1,"saturation":220.3,"saturationExtrapolated":true,"frontierPct":84.5,"extrapolated":false,"extrapolation":null,"maxTested":123.9,"saturationDate":"2029-12-09","saturationDateLo":"2028-12-06","saturationDateHi":"2031-03-06","saturated":false},{"slug":"mbpp-plus","family":"mbpp-plus","name":"MBPP+","nStates":104,"nModels":104,"difficulty":89.4,"saturation":189.6,"saturationExtrapolated":true,"frontierPct":85.3,"extrapolated":false,"extrapolation":null,"maxTested":151.8,"saturationDate":"2027-12-12","saturationDateLo":"2027-04-30","saturationDateHi":"2028-10-09","saturated":false},{"slug":"commonsenseqa","family":"commonsenseqa","name":"CommonSenseQA","nStates":18,"nModels":18,"difficulty":89.5,"saturation":127.3,"saturationExtrapolated":false,"frontierPct":99.3,"extrapolated":false,"extrapolation":null,"maxTested":120.3,"saturationDate":"2024-04-09","saturationDateLo":"2023-06-13","saturationDateHi":"2024-09-12","saturated":true},{"slug":"mvbench","family":"mvbench","name":"MVBench","nStates":51,"nModels":47,"difficulty":89.9,"saturation":216.7,"saturationExtrapolated":true,"frontierPct":80.7,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2029-07-14","saturationDateLo":"2028-09-06","saturationDateHi":"2030-09-23","saturated":false},{"slug":"ifeval","family":"ifeval","name":"IFEval","nStates":167,"nModels":145,"difficulty":90.3,"saturation":144.8,"saturationExtrapolated":false,"frontierPct":95.8,"extrapolated":false,"extrapolation":null,"maxTested":155.9,"saturationDate":"2025-03-24","saturationDateLo":"2025-03-24","saturationDateHi":"2025-04-15","saturated":true},{"slug":"cv-bench","family":"cv-bench","name":"CV-Bench","nStates":9,"nModels":8,"difficulty":90.8,"saturation":194.1,"saturationExtrapolated":true,"frontierPct":92.2,"extrapolated":true,"extrapolation":"below","maxTested":148.4,"saturationDate":"2027-11-29","saturationDateLo":"2026-04-22","saturationDateHi":"2030-03-14","saturated":false},{"slug":"creative-writing-v2","family":"creative-writing-v2","name":"Creative Writing v2","nStates":53,"nModels":53,"difficulty":90.9,"saturation":154.6,"saturationExtrapolated":false,"frontierPct":93.5,"extrapolated":false,"extrapolation":null,"maxTested":145.5,"saturationDate":"2025-11-18","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-11","saturated":true},{"slug":"eq-bench-v2","family":"eq-bench-v2","name":"EQ-Bench v2","nStates":98,"nModels":98,"difficulty":91.0,"saturation":135.4,"saturationExtrapolated":false,"frontierPct":97.8,"extrapolated":false,"extrapolation":null,"maxTested":137.5,"saturationDate":"2024-09-12","saturationDateLo":"2024-08-29","saturationDateHi":"2024-12-04","saturated":true},{"slug":"quality","family":"quality","name":"QuALITY","nStates":8,"nModels":8,"difficulty":91.1,"saturation":134.4,"saturationExtrapolated":false,"frontierPct":98.9,"extrapolated":true,"extrapolation":"below","maxTested":123.7,"saturationDate":"2024-09-12","saturationDateLo":"2024-05-13","saturationDateHi":"2024-12-17","saturated":true},{"slug":"gsm8k","family":"gsm8k","name":"GSM8K","nStates":212,"nModels":210,"difficulty":91.6,"saturation":122.2,"saturationExtrapolated":false,"frontierPct":99.6,"extrapolated":false,"extrapolation":null,"maxTested":152.0,"saturationDate":"2023-11-05","saturationDateLo":"2023-04-18","saturationDateHi":"2023-11-05","saturated":true},{"slug":"infographicvqa-val","family":"infographicvqa-val","name":"InfographicVQA (val)","nStates":14,"nModels":13,"difficulty":91.6,"saturation":145.6,"saturationExtrapolated":false,"frontierPct":96.3,"extrapolated":false,"extrapolation":null,"maxTested":144.7,"saturationDate":"2025-03-24","saturationDateLo":"2024-09-12","saturationDateHi":"2025-06-09","saturated":true},{"slug":"vstar-bench","family":"vstar-bench","name":"V* Bench","nStates":27,"nModels":25,"difficulty":91.7,"saturation":172.5,"saturationExtrapolated":true,"frontierPct":90.2,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2026-09-02","saturationDateLo":"2026-02-11","saturationDateHi":"2027-09-16","saturated":true},{"slug":"alignbench-v1-1","family":"alignbench-v1-1","name":"AlignBench v1.1","nStates":48,"nModels":40,"difficulty":91.8,"saturation":152.5,"saturationExtrapolated":false,"frontierPct":94.0,"extrapolated":false,"extrapolation":null,"maxTested":140.8,"saturationDate":"2025-07-31","saturationDateLo":"2025-06-09","saturationDateHi":"2025-12-11","saturated":true},{"slug":"screenspot-v2-desktop","family":"screenspot-v2","name":"ScreenSpot-v2","nStates":9,"nModels":8,"difficulty":92.1,"saturation":143.5,"saturationExtrapolated":false,"frontierPct":97.2,"extrapolated":true,"extrapolation":"below","maxTested":152.8,"saturationDate":"2024-12-17","saturationDateLo":"2024-09-12","saturationDateHi":"2025-12-11","saturated":true},{"slug":"mmlu-medical-genetics-openmedical-harness","family":"mmlu-medical-genetics-openmedical-harness","name":"MMLU Medical Genetics (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":93.1,"saturation":115.5,"saturationExtrapolated":false,"frontierPct":100.0,"extrapolated":false,"extrapolation":null,"maxTested":103.7,"saturationDate":"2023-03-13","saturationDateLo":"2023-03-13","saturationDateHi":"2023-03-13","saturated":true},{"slug":"arc-challenge","family":"arc-ai2","name":"ARC AI2","nStates":101,"nModels":101,"difficulty":93.6,"saturation":122.2,"saturationExtrapolated":false,"frontierPct":99.8,"extrapolated":false,"extrapolation":null,"maxTested":130.0,"saturationDate":"2023-11-05","saturationDateLo":"2023-03-13","saturationDateHi":"2024-01-24","saturated":true},{"slug":"alignbench","family":"alignbench","name":"AlignBench (v1.0 or version unstated)","nStates":21,"nModels":21,"difficulty":94.0,"saturation":145.0,"saturationExtrapolated":false,"frontierPct":96.4,"extrapolated":true,"extrapolation":"below","maxTested":122.8,"saturationDate":"2025-03-24","saturationDateLo":"2024-09-12","saturationDateHi":"2025-07-31","saturated":true},{"slug":"mmlu-college-biology-openmedical-harness","family":"mmlu-college-biology-openmedical-harness","name":"MMLU College Biology (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":94.2,"saturation":115.6,"saturationExtrapolated":false,"frontierPct":100.0,"extrapolated":false,"extrapolation":null,"maxTested":103.7,"saturationDate":"2023-03-13","saturationDateLo":"2023-03-13","saturationDateHi":"2023-03-13","saturated":true},{"slug":"mmlu-clinical-knowledge-openmedical-harness","family":"mmlu-clinical-knowledge-openmedical-harness","name":"MMLU Clinical Knowledge (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":94.8,"saturation":119.7,"saturationExtrapolated":false,"frontierPct":99.9,"extrapolated":false,"extrapolation":null,"maxTested":103.7,"saturationDate":"2023-03-13","saturationDateLo":"2023-03-13","saturationDateHi":"2023-11-05","saturated":true},{"slug":"medic-closed-ended","family":"medic-closed-ended","name":"MEDIC (closed-ended)","nStates":43,"nModels":43,"difficulty":95.1,"saturation":211.1,"saturationExtrapolated":true,"frontierPct":81.1,"extrapolated":false,"extrapolation":null,"maxTested":146.0,"saturationDate":"2029-02-17","saturationDateLo":"2027-11-15","saturationDateHi":"2031-02-13","saturated":false},{"slug":"cluewsc","family":"cluewsc","name":"CLUEWSC","nStates":30,"nModels":30,"difficulty":95.2,"saturation":164.1,"saturationExtrapolated":false,"frontierPct":95.9,"extrapolated":false,"extrapolation":null,"maxTested":139.4,"saturationDate":"2026-02-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-12-02","saturated":true},{"slug":"mbpp","family":"mbpp","name":"MBPP","nStates":180,"nModels":177,"difficulty":95.5,"saturation":143.6,"saturationExtrapolated":false,"frontierPct":96.7,"extrapolated":false,"extrapolation":null,"maxTested":139.9,"saturationDate":"2025-03-24","saturationDateLo":"2024-09-12","saturationDateHi":"2025-04-03","saturated":true},{"slug":"toolsandbox","family":"toolsandbox","name":"ToolSandbox","nStates":8,"nModels":8,"difficulty":95.5,"saturation":179.9,"saturationExtrapolated":true,"frontierPct":89.0,"extrapolated":true,"extrapolation":"below","maxTested":147.7,"saturationDate":"2027-04-02","saturationDateLo":"2026-02-11","saturationDateHi":"2028-08-22","saturated":false},{"slug":"gsm1k","family":"gsm1k","name":"GSM1k","nStates":16,"nModels":16,"difficulty":96.4,"saturation":118.5,"saturationExtrapolated":false,"frontierPct":99.9,"extrapolated":false,"extrapolation":null,"maxTested":130.0,"saturationDate":"2023-03-13","saturationDateLo":"2023-03-13","saturationDateHi":"2023-06-13","saturated":true},{"slug":"slake","family":"slake","name":"SLAKE","nStates":25,"nModels":23,"difficulty":97.0,"saturation":184.1,"saturationExtrapolated":true,"frontierPct":87.6,"extrapolated":true,"extrapolation":"below","maxTested":161.6,"saturationDate":"2027-06-23","saturationDateLo":"2026-04-07","saturationDateHi":"2028-05-05","saturated":false},{"slug":"mmlu-professional-medicine-openmedical-harness","family":"mmlu-professional-medicine-openmedical-harness","name":"MMLU Professional Medicine (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":97.2,"saturation":115.9,"saturationExtrapolated":false,"frontierPct":100.0,"extrapolated":false,"extrapolation":null,"maxTested":103.7,"saturationDate":"2023-03-13","saturationDateLo":"2023-03-13","saturationDateHi":"2023-06-13","saturated":true},{"slug":"mlvu-m-avg","family":"mlvu-m-avg","name":"MLVU (multiple-choice M-Avg)","nStates":69,"nModels":58,"difficulty":97.7,"saturation":172.6,"saturationExtrapolated":true,"frontierPct":89.6,"extrapolated":true,"extrapolation":"below","maxTested":162.2,"saturationDate":"2026-09-02","saturationDateLo":"2026-04-22","saturationDateHi":"2027-03-12","saturated":true},{"slug":"api-bank-meta-corrected","family":"api-bank-meta-corrected","name":"API-Bank (Meta corrected protocol)","nStates":9,"nModels":9,"difficulty":98.1,"saturation":129.1,"saturationExtrapolated":false,"frontierPct":99.6,"extrapolated":true,"extrapolation":"below","maxTested":130.0,"saturationDate":"2024-05-13","saturationDateLo":"2023-11-05","saturationDateHi":"2024-09-12","saturated":true},{"slug":"winogrande-openllm-v1-harness","family":"winogrande-openllm-v1-harness","name":"WinoGrande (Open LLM Leaderboard v1)","nStates":115,"nModels":115,"difficulty":98.4,"saturation":156.2,"saturationExtrapolated":false,"frontierPct":96.8,"extrapolated":false,"extrapolation":null,"maxTested":121.7,"saturationDate":"2025-12-11","saturationDateLo":"2025-06-09","saturationDateHi":"2026-04-07","saturated":true},{"slug":"global-piqa","family":"global-piqa","name":"Global PIQA","nStates":35,"nModels":33,"difficulty":98.6,"saturation":151.6,"saturationExtrapolated":false,"frontierPct":94.6,"extrapolated":true,"extrapolation":"below","maxTested":156.0,"saturationDate":"2025-07-31","saturationDateLo":"2025-07-31","saturationDateHi":"2025-10-06","saturated":true},{"slug":"multipl-e-humaneval-csharp","family":"multipl-e-humaneval-csharp","name":"MultiPL-E HumanEval (C#)","nStates":51,"nModels":51,"difficulty":98.7,"saturation":171.8,"saturationExtrapolated":true,"frontierPct":89.8,"extrapolated":false,"extrapolation":null,"maxTested":138.7,"saturationDate":"2026-09-02","saturationDateLo":"2025-09-29","saturationDateHi":"2027-12-28","saturated":true},{"slug":"mmlu-openllm-v1-harness","family":"mmlu-openllm-v1-harness","name":"MMLU (Open LLM Leaderboard v1)","nStates":117,"nModels":117,"difficulty":98.9,"saturation":135.4,"saturationExtrapolated":false,"frontierPct":98.9,"extrapolated":false,"extrapolation":null,"maxTested":121.7,"saturationDate":"2024-09-12","saturationDateLo":"2024-08-29","saturationDateHi":"2024-11-21","saturated":true},{"slug":"morebench","family":"morebench","name":"MoReBench","nStates":14,"nModels":14,"difficulty":99.1,"saturation":199.8,"saturationExtrapolated":true,"frontierPct":85.1,"extrapolated":true,"extrapolation":"below","maxTested":152.3,"saturationDate":"2028-03-10","saturationDateLo":"2027-03-15","saturationDateHi":"2029-08-20","saturated":false},{"slug":"cmmlu","family":"cmmlu","name":"CMMLU","nStates":55,"nModels":55,"difficulty":99.4,"saturation":140.5,"saturationExtrapolated":false,"frontierPct":98.3,"extrapolated":false,"extrapolation":null,"maxTested":132.0,"saturationDate":"2024-12-04","saturationDateLo":"2024-09-12","saturationDateHi":"2025-04-15","saturated":true},{"slug":"humaneval","family":"humaneval","name":"HumanEval","nStates":204,"nModels":196,"difficulty":99.6,"saturation":139.8,"saturationExtrapolated":false,"frontierPct":97.7,"extrapolated":false,"extrapolation":null,"maxTested":148.6,"saturationDate":"2024-12-04","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"frames","family":"frames","name":"FRAMES","nStates":17,"nModels":17,"difficulty":99.7,"saturation":164.6,"saturationExtrapolated":false,"frontierPct":92.6,"extrapolated":true,"extrapolation":"below","maxTested":147.5,"saturationDate":"2026-03-05","saturationDateLo":"2025-07-31","saturationDateHi":"2027-03-14","saturated":true},{"slug":"mmlu","family":"mmlu","name":"MMLU","nStates":280,"nModels":267,"difficulty":99.8,"saturation":146.2,"saturationExtrapolated":false,"frontierPct":97.1,"extrapolated":false,"extrapolation":null,"maxTested":147.9,"saturationDate":"2025-04-15","saturationDateLo":"2025-03-24","saturationDateHi":"2025-04-15","saturated":true},{"slug":"mmlu-college-medicine-openmedical-harness","family":"mmlu-college-medicine-openmedical-harness","name":"MMLU College Medicine (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":99.9,"saturation":124.7,"saturationExtrapolated":false,"frontierPct":99.8,"extrapolated":false,"extrapolation":null,"maxTested":103.7,"saturationDate":"2024-01-24","saturationDateLo":"2023-06-13","saturationDateHi":"2024-05-13","saturated":true},{"slug":"pubmedqa","family":"pubmedqa","name":"PubMedQA","nStates":9,"nModels":9,"difficulty":100.2,"saturation":237.4,"saturationExtrapolated":true,"frontierPct":84.3,"extrapolated":true,"extrapolation":"below","maxTested":147.1,"saturationDate":"2031-01-11","saturationDateLo":"2028-11-30","saturationDateHi":"2034-02-11","saturated":false},{"slug":"gsm1k-lm-eval-harness","family":"gsm1k-lm-eval-harness","name":"GSM1k (paper LM Evaluation Harness protocol)","nStates":46,"nModels":46,"difficulty":100.3,"saturation":124.4,"saturationExtrapolated":false,"frontierPct":99.8,"extrapolated":false,"extrapolation":null,"maxTested":123.7,"saturationDate":"2023-11-05","saturationDateLo":"2023-06-13","saturationDateHi":"2024-05-13","saturated":true},{"slug":"charxiv-descriptive","family":"charxiv-descriptive","name":"CharXiv-D","nStates":72,"nModels":64,"difficulty":100.6,"saturation":142.9,"saturationExtrapolated":false,"frontierPct":97.4,"extrapolated":false,"extrapolation":null,"maxTested":151.1,"saturationDate":"2024-12-17","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"embspatial-bench","family":"embspatial-bench","name":"EmbSpatial-Bench","nStates":33,"nModels":29,"difficulty":100.6,"saturation":202.2,"saturationExtrapolated":true,"frontierPct":86.6,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2028-10-01","saturationDateLo":"2027-11-22","saturationDateHi":"2030-02-06","saturated":false},{"slug":"mmvet-vlmevalkit","family":"mmvet-vlmevalkit","name":"MM-Vet (OpenVLM)","nStates":57,"nModels":57,"difficulty":100.6,"saturation":171.3,"saturationExtrapolated":true,"frontierPct":89.4,"extrapolated":false,"extrapolation":null,"maxTested":149.3,"saturationDate":"2026-10-10","saturationDateLo":"2026-04-07","saturationDateHi":"2027-04-18","saturated":true},{"slug":"humaneval-plus","family":"humaneval-plus","name":"HumanEval+","nStates":112,"nModels":112,"difficulty":100.7,"saturation":159.6,"saturationExtrapolated":false,"frontierPct":92.8,"extrapolated":false,"extrapolation":null,"maxTested":139.3,"saturationDate":"2026-02-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-07-10","saturated":true},{"slug":"vlmsareblind","family":"vlmsareblind","name":"VLMs Are Blind","nStates":19,"nModels":17,"difficulty":100.8,"saturation":150.8,"saturationExtrapolated":false,"frontierPct":95.7,"extrapolated":true,"extrapolation":"below","maxTested":149.6,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-12-11","saturated":true},{"slug":"formationeval","family":"formationeval","name":"FormationEval","nStates":67,"nModels":57,"difficulty":100.9,"saturation":136.0,"saturationExtrapolated":false,"frontierPct":98.9,"extrapolated":true,"extrapolation":"below","maxTested":153.7,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2024-12-04","saturated":true},{"slug":"crm-agent-bench","family":"crm-agent-bench","name":"CRM-Agent-Bench","nStates":14,"nModels":14,"difficulty":101.0,"saturation":184.4,"saturationExtrapolated":true,"frontierPct":85.7,"extrapolated":true,"extrapolation":"below","maxTested":162.3,"saturationDate":"2027-10-11","saturationDateLo":"2027-03-31","saturationDateHi":"2028-05-05","saturated":false},{"slug":"mgsm","family":"mgsm","name":"MGSM","nStates":136,"nModels":116,"difficulty":101.4,"saturation":136.9,"saturationExtrapolated":false,"frontierPct":98.4,"extrapolated":false,"extrapolation":null,"maxTested":154.5,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2024-12-04","saturated":true},{"slug":"mme","family":"mme","name":"MME","nStates":23,"nModels":22,"difficulty":101.4,"saturation":169.1,"saturationExtrapolated":false,"frontierPct":94.4,"extrapolated":false,"extrapolation":null,"maxTested":148.5,"saturationDate":"2026-07-14","saturationDateLo":"2025-07-31","saturationDateHi":"2027-08-31","saturated":true},{"slug":"regretbench-success","family":"regretbench-success","name":"RegretBench (goal success)","nStates":14,"nModels":12,"difficulty":101.6,"saturation":186.5,"saturationExtrapolated":true,"frontierPct":87.0,"extrapolated":true,"extrapolation":"below","maxTested":156.8,"saturationDate":"2027-08-29","saturationDateLo":"2026-09-02","saturationDateHi":"2029-04-03","saturated":false},{"slug":"bbh-openllm-harness","family":"bbh-openllm-harness","name":"BBH (Open LLM Leaderboard v2)","nStates":121,"nModels":121,"difficulty":101.7,"saturation":179.4,"saturationExtrapolated":true,"frontierPct":86.9,"extrapolated":false,"extrapolation":null,"maxTested":126.1,"saturationDate":"2027-05-25","saturationDateLo":"2026-12-27","saturationDateHi":"2027-09-21","saturated":false},{"slug":"gsm8k-openllm-v1-harness","family":"gsm8k-openllm-v1-harness","name":"GSM8K (Open LLM Leaderboard v1)","nStates":91,"nModels":91,"difficulty":102.0,"saturation":141.9,"saturationExtrapolated":false,"frontierPct":97.6,"extrapolated":false,"extrapolation":null,"maxTested":121.7,"saturationDate":"2024-12-04","saturationDateLo":"2024-09-12","saturationDateHi":"2025-04-21","saturated":true},{"slug":"tau3-bench","family":"tau3-bench","name":"\u03c4\u00b3-bench average","nStates":20,"nModels":20,"difficulty":102.3,"saturation":266.8,"saturationExtrapolated":true,"frontierPct":72.1,"extrapolated":true,"extrapolation":"below","maxTested":162.2,"saturationDate":"2032-11-11","saturationDateLo":"2030-08-21","saturationDateHi":"2035-07-14","saturated":false},{"slug":"winogrande","family":"winogrande","name":"WinoGrande","nStates":121,"nModels":121,"difficulty":102.3,"saturation":166.4,"saturationExtrapolated":false,"frontierPct":95.3,"extrapolated":false,"extrapolation":null,"maxTested":132.0,"saturationDate":"2026-07-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-12-04","saturated":true},{"slug":"multipl-e-humaneval-8-language-average","family":"multipl-e-humaneval-8-language-average","name":"MultiPL-E (HumanEval, 8-language average)","nStates":84,"nModels":84,"difficulty":102.4,"saturation":164.2,"saturationExtrapolated":false,"frontierPct":91.5,"extrapolated":false,"extrapolation":null,"maxTested":139.4,"saturationDate":"2026-04-07","saturationDateLo":"2025-11-16","saturationDateHi":"2026-11-20","saturated":true},{"slug":"tax_eval_v2","family":"tax_eval_v2","name":"TaxEval v2","nStates":136,"nModels":126,"difficulty":102.5,"saturation":208.1,"saturationExtrapolated":true,"frontierPct":79.5,"extrapolated":false,"extrapolation":null,"maxTested":166.6,"saturationDate":"2029-04-19","saturationDateLo":"2028-12-04","saturationDateHi":"2029-10-09","saturated":false},{"slug":"mmlu-anatomy-openmedical-harness","family":"mmlu-anatomy-openmedical-harness","name":"MMLU Anatomy (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":102.7,"saturation":133.4,"saturationExtrapolated":false,"frontierPct":99.4,"extrapolated":false,"extrapolation":null,"maxTested":103.7,"saturationDate":"2024-09-12","saturationDateLo":"2024-04-09","saturationDateHi":"2025-03-24","saturated":true},{"slug":"multipl-e-humaneval-java","family":"multipl-e-humaneval-java","name":"MultiPL-E HumanEval (Java)","nStates":18,"nModels":18,"difficulty":102.8,"saturation":150.3,"saturationExtrapolated":false,"frontierPct":95.9,"extrapolated":false,"extrapolation":null,"maxTested":125.8,"saturationDate":"2025-04-15","saturationDateLo":"2024-09-12","saturationDateHi":"2026-02-11","saturated":true},{"slug":"arc-challenge-openllm-v1-harness","family":"arc-challenge-openllm-v1-harness","name":"ARC-Challenge (Open LLM Leaderboard v1)","nStates":118,"nModels":118,"difficulty":103.2,"saturation":157.7,"saturationExtrapolated":false,"frontierPct":95.0,"extrapolated":false,"extrapolation":null,"maxTested":121.7,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-04-07","saturated":true},{"slug":"blxbench","family":"blxbench","name":"BLXBench","nStates":29,"nModels":29,"difficulty":103.3,"saturation":241.5,"saturationExtrapolated":true,"frontierPct":75.3,"extrapolated":true,"extrapolation":"below","maxTested":165.5,"saturationDate":"2031-02-04","saturationDateLo":"2029-05-21","saturationDateHi":"2034-01-20","saturated":false},{"slug":"mmlu-redux","family":"mmlu-redux","name":"MMLU-Redux","nStates":190,"nModels":173,"difficulty":103.7,"saturation":141.6,"saturationExtrapolated":false,"frontierPct":98.2,"extrapolated":false,"extrapolation":null,"maxTested":156.1,"saturationDate":"2024-12-17","saturationDateLo":"2024-11-21","saturationDateHi":"2025-03-24","saturated":true},{"slug":"c-eval","family":"c-eval","name":"C-Eval","nStates":162,"nModels":149,"difficulty":103.8,"saturation":156.7,"saturationExtrapolated":false,"frontierPct":95.1,"extrapolated":false,"extrapolation":null,"maxTested":153.6,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-14","saturated":true},{"slug":"multipl-e-humaneval-cpp","family":"multipl-e-humaneval-cpp","name":"MultiPL-E HumanEval (C++)","nStates":18,"nModels":18,"difficulty":103.9,"saturation":150.2,"saturationExtrapolated":false,"frontierPct":96.1,"extrapolated":false,"extrapolation":null,"maxTested":125.8,"saturationDate":"2025-04-15","saturationDateLo":"2024-09-12","saturationDateHi":"2026-06-09","saturated":true},{"slug":"mathvista-vlmevalkit","family":"mathvista-vlmevalkit","name":"MathVista (OpenVLM)","nStates":58,"nModels":58,"difficulty":104.1,"saturation":175.1,"saturationExtrapolated":true,"frontierPct":88.2,"extrapolated":false,"extrapolation":null,"maxTested":149.3,"saturationDate":"2027-01-23","saturationDateLo":"2026-09-02","saturationDateHi":"2027-07-23","saturated":false},{"slug":"writingbench","family":"writingbench","name":"WritingBench","nStates":72,"nModels":64,"difficulty":104.4,"saturation":164.1,"saturationExtrapolated":false,"frontierPct":91.5,"extrapolated":false,"extrapolation":null,"maxTested":147.5,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-02","saturated":true},{"slug":"ruler-128k","family":"ruler-128k","name":"RULER - 128K","nStates":26,"nModels":26,"difficulty":105.0,"saturation":159.3,"saturationExtrapolated":false,"frontierPct":95.0,"extrapolated":true,"extrapolation":"below","maxTested":142.2,"saturationDate":"2025-12-11","saturationDateLo":"2025-03-24","saturationDateHi":"2027-02-26","saturated":true},{"slug":"bbh","family":"bbh","name":"BBH (BIG-Bench Hard)","nStates":109,"nModels":109,"difficulty":105.1,"saturation":134.6,"saturationExtrapolated":false,"frontierPct":99.3,"extrapolated":false,"extrapolation":null,"maxTested":133.6,"saturationDate":"2024-09-12","saturationDateLo":"2024-08-29","saturationDateHi":"2024-09-12","saturated":true},{"slug":"ifeval-openllm-harness","family":"ifeval-openllm-harness","name":"IFEval (Open LLM Leaderboard v2)","nStates":121,"nModels":121,"difficulty":105.2,"saturation":172.7,"saturationExtrapolated":true,"frontierPct":89.0,"extrapolated":false,"extrapolation":null,"maxTested":126.1,"saturationDate":"2026-09-02","saturationDateLo":"2026-02-11","saturationDateHi":"2027-10-22","saturated":true},{"slug":"medqa-4options-openmedical-harness","family":"medqa-4options-openmedical-harness","name":"MedQA 4-options (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":105.2,"saturation":129.2,"saturationExtrapolated":true,"frontierPct":99.8,"extrapolated":true,"extrapolation":"above","maxTested":103.7,"saturationDate":"2024-05-13","saturationDateLo":"2024-01-24","saturationDateHi":"2024-09-12","saturated":true},{"slug":"multipl-e-humaneval-php","family":"multipl-e-humaneval-php","name":"MultiPL-E HumanEval (PHP)","nStates":18,"nModels":18,"difficulty":105.5,"saturation":151.0,"saturationExtrapolated":false,"frontierPct":95.8,"extrapolated":false,"extrapolation":null,"maxTested":125.8,"saturationDate":"2025-04-15","saturationDateLo":"2024-11-21","saturationDateHi":"2026-02-11","saturated":true},{"slug":"dynamath","family":"dynamath","name":"DynaMath","nStates":33,"nModels":29,"difficulty":107.1,"saturation":159.6,"saturationExtrapolated":false,"frontierPct":93.5,"extrapolated":false,"extrapolation":null,"maxTested":152.7,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-11","saturated":true},{"slug":"blink","family":"blink","name":"BLINK","nStates":51,"nModels":43,"difficulty":107.2,"saturation":210.0,"saturationExtrapolated":true,"frontierPct":79.9,"extrapolated":false,"extrapolation":null,"maxTested":156.9,"saturationDate":"2029-02-13","saturationDateLo":"2028-08-06","saturationDateHi":"2029-11-24","saturated":false},{"slug":"mm-ifeval","family":"mm-ifeval","name":"MM-IFEval","nStates":15,"nModels":14,"difficulty":107.7,"saturation":177.5,"saturationExtrapolated":true,"frontierPct":87.8,"extrapolated":false,"extrapolation":null,"maxTested":150.0,"saturationDate":"2027-02-20","saturationDateLo":"2026-04-07","saturationDateHi":"2028-02-21","saturated":false},{"slug":"geobench","family":"geobench","name":"GeoBench","nStates":39,"nModels":38,"difficulty":107.8,"saturation":195.6,"saturationExtrapolated":true,"frontierPct":83.1,"extrapolated":true,"extrapolation":"below","maxTested":165.3,"saturationDate":"2028-03-27","saturationDateLo":"2027-07-04","saturationDateHi":"2029-05-14","saturated":false},{"slug":"race-high","family":"race-high","name":"RACE-High","nStates":33,"nModels":33,"difficulty":108.1,"saturation":178.0,"saturationExtrapolated":true,"frontierPct":91.3,"extrapolated":false,"extrapolation":null,"maxTested":123.9,"saturationDate":"2026-12-28","saturationDateLo":"2025-11-13","saturationDateHi":"2029-04-27","saturated":false},{"slug":"mmmu-val-vlmevalkit","family":"mmmu-val-vlmevalkit","name":"MMMU Validation (OpenVLM)","nStates":58,"nModels":58,"difficulty":108.3,"saturation":182.0,"saturationExtrapolated":true,"frontierPct":85.7,"extrapolated":false,"extrapolation":null,"maxTested":149.3,"saturationDate":"2027-08-05","saturationDateLo":"2027-03-13","saturationDateHi":"2027-12-19","saturated":false},{"slug":"mm-mt-bench","family":"mm-mt-bench","name":"MM-MT-Bench","nStates":21,"nModels":21,"difficulty":108.5,"saturation":158.2,"saturationExtrapolated":false,"frontierPct":94.0,"extrapolated":false,"extrapolation":null,"maxTested":133.6,"saturationDate":"2025-12-11","saturationDateLo":"2025-04-15","saturationDateHi":"2026-07-27","saturated":true},{"slug":"muirbench","family":"muirbench","name":"MuirBench","nStates":33,"nModels":27,"difficulty":108.7,"saturation":181.6,"saturationExtrapolated":true,"frontierPct":87.4,"extrapolated":false,"extrapolation":null,"maxTested":151.7,"saturationDate":"2027-02-19","saturationDateLo":"2026-09-02","saturationDateHi":"2027-09-22","saturated":false},{"slug":"multipl-e-humaneval-typescript","family":"multipl-e-humaneval-typescript","name":"MultiPL-E HumanEval (TypeScript)","nStates":18,"nModels":18,"difficulty":108.9,"saturation":160.2,"saturationExtrapolated":false,"frontierPct":93.7,"extrapolated":false,"extrapolation":null,"maxTested":125.8,"saturationDate":"2025-12-11","saturationDateLo":"2025-03-24","saturationDateHi":"2027-05-28","saturated":true},{"slug":"aider-code-editing","family":"aider-code-editing","name":"Aider Code Editing","nStates":8,"nModels":8,"difficulty":109.1,"saturation":142.5,"saturationExtrapolated":false,"frontierPct":98.2,"extrapolated":false,"extrapolation":null,"maxTested":133.6,"saturationDate":"2024-12-17","saturationDateLo":"2024-09-12","saturationDateHi":"2025-04-15","saturated":true},{"slug":"cruxeval-o","family":"cruxeval-o","name":"CRUXEval-O","nStates":21,"nModels":21,"difficulty":109.4,"saturation":141.4,"saturationExtrapolated":false,"frontierPct":98.5,"extrapolated":false,"extrapolation":null,"maxTested":129.6,"saturationDate":"2024-12-04","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-25","saturated":true},{"slug":"concept-synth-abd-v1-2","family":"concept-synth-abd-v1-2","name":"Concept Synth ABD v1.2","nStates":9,"nModels":9,"difficulty":110.3,"saturation":151.8,"saturationExtrapolated":false,"frontierPct":97.2,"extrapolated":true,"extrapolation":"below","maxTested":156.3,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2026-02-11","saturated":true},{"slug":"longvideobench-reported","family":"longvideobench-reported","name":"LongVideoBench - release-reported accuracy","nStates":12,"nModels":11,"difficulty":110.3,"saturation":197.5,"saturationExtrapolated":true,"frontierPct":81.8,"extrapolated":false,"extrapolation":null,"maxTested":155.4,"saturationDate":"2028-06-17","saturationDateLo":"2027-09-21","saturationDateHi":"2029-07-29","saturated":false},{"slug":"medmcqa-openmedical-harness","family":"medmcqa-openmedical-harness","name":"MedMCQA (Open Medical-LLM)","nStates":17,"nModels":17,"difficulty":110.5,"saturation":144.9,"saturationExtrapolated":true,"frontierPct":98.4,"extrapolated":true,"extrapolation":"above","maxTested":103.7,"saturationDate":"2025-03-24","saturationDateLo":"2024-09-12","saturationDateHi":"2025-12-11","saturated":true},{"slug":"cruxeval-i","family":"cruxeval-i","name":"CRUXEval-I","nStates":21,"nModels":21,"difficulty":110.8,"saturation":147.7,"saturationExtrapolated":false,"frontierPct":97.0,"extrapolated":false,"extrapolation":null,"maxTested":129.6,"saturationDate":"2025-04-15","saturationDateLo":"2024-12-17","saturationDateHi":"2025-07-31","saturated":true},{"slug":"medqa","family":"medqa","name":"MedQA","nStates":71,"nModels":58,"difficulty":111.1,"saturation":154.1,"saturationExtrapolated":false,"frontierPct":96.3,"extrapolated":false,"extrapolation":null,"maxTested":155.7,"saturationDate":"2025-07-31","saturationDateLo":"2025-07-31","saturationDateHi":"2025-12-16","saturated":true},{"slug":"openeqa-em-eqa","family":"openeqa-em-eqa","name":"OpenEQA (episodic memory)","nStates":9,"nModels":8,"difficulty":111.3,"saturation":187.6,"saturationExtrapolated":true,"frontierPct":85.7,"extrapolated":true,"extrapolation":"below","maxTested":152.5,"saturationDate":"2027-09-05","saturationDateLo":"2026-09-02","saturationDateHi":"2029-06-27","saturated":false},{"slug":"math","family":"math","name":"MATH","nStates":201,"nModels":201,"difficulty":111.5,"saturation":144.6,"saturationExtrapolated":false,"frontierPct":97.8,"extrapolated":false,"extrapolation":null,"maxTested":133.6,"saturationDate":"2025-03-24","saturationDateLo":"2024-12-04","saturationDateHi":"2025-04-15","saturated":true},{"slug":"zeroeval-crux","family":"zeroeval-crux","name":"ZeroEval CRUX","nStates":41,"nModels":41,"difficulty":111.5,"saturation":139.5,"saturationExtrapolated":false,"frontierPct":99.3,"extrapolated":false,"extrapolation":null,"maxTested":146.6,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2024-09-12","saturated":true},{"slug":"btf3-2386-pooled","family":"btf3-2386-pooled","name":"Bench to the Future 3 \u2014 September Panel \u2014 Pooled","nStates":8,"nModels":8,"difficulty":112.0,"saturation":171.1,"saturationExtrapolated":true,"frontierPct":91.0,"extrapolated":true,"extrapolation":"below","maxTested":164.6,"saturationDate":"2026-11-11","saturationDateLo":"2026-07-20","saturationDateHi":"2027-03-15","saturated":false},{"slug":"multi-if","family":"multi-if","name":"Multi-IF","nStates":89,"nModels":78,"difficulty":112.0,"saturation":169.2,"saturationExtrapolated":true,"frontierPct":89.4,"extrapolated":false,"extrapolation":null,"maxTested":147.7,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2027-02-05","saturated":true},{"slug":"hallusionbench","family":"hallusionbench-vlmevalkit","name":"HallusionBench (OpenVLM)","nStates":57,"nModels":46,"difficulty":112.7,"saturation":226.4,"saturationExtrapolated":true,"frontierPct":76.5,"extrapolated":false,"extrapolation":null,"maxTested":152.7,"saturationDate":"2030-01-12","saturationDateLo":"2029-05-31","saturationDateHi":"2030-10-31","saturated":false},{"slug":"bfcl-v3-overall-accuracy","family":"bfcl-v3-overall-accuracy","name":"BFCL v3 Overall Accuracy","nStates":94,"nModels":83,"difficulty":113.1,"saturation":188.0,"saturationExtrapolated":true,"frontierPct":83.4,"extrapolated":false,"extrapolation":null,"maxTested":148.2,"saturationDate":"2027-12-28","saturationDateLo":"2027-08-18","saturationDateHi":"2028-05-23","saturated":false},{"slug":"medqa-vals-demographic-bias","family":"medqa-vals-demographic-bias","name":"MedQA Demographic Bias (Vals)","nStates":90,"nModels":80,"difficulty":113.2,"saturation":147.4,"saturationExtrapolated":false,"frontierPct":98.0,"extrapolated":false,"extrapolation":null,"maxTested":161.1,"saturationDate":"2025-03-24","saturationDateLo":"2025-03-24","saturationDateHi":"2025-04-15","saturated":true},{"slug":"egoschema","family":"egoschema","name":"EgoSchema","nStates":18,"nModels":18,"difficulty":113.3,"saturation":180.6,"saturationExtrapolated":true,"frontierPct":90.2,"extrapolated":false,"extrapolation":null,"maxTested":133.5,"saturationDate":"2027-03-28","saturationDateLo":"2026-03-05","saturationDateHi":"2029-04-08","saturated":false},{"slug":"ruby-llm-program-fixer","family":"ruby-llm-program-fixer","name":"Ruby LLM Benchmarks \u2014 Program Fixer","nStates":215,"nModels":111,"difficulty":113.3,"saturation":232.0,"saturationExtrapolated":true,"frontierPct":73.5,"extrapolated":false,"extrapolation":null,"maxTested":167.4,"saturationDate":"2030-11-08","saturationDateLo":"2030-03-24","saturationDateHi":"2031-07-04","saturated":false},{"slug":"activitynet-qa","family":"activitynet-qa","name":"ActivityNet-QA","nStates":18,"nModels":17,"difficulty":113.7,"saturation":218.9,"saturationExtrapolated":true,"frontierPct":77.1,"extrapolated":true,"extrapolation":"below","maxTested":150.8,"saturationDate":"2029-08-12","saturationDateLo":"2028-05-16","saturationDateHi":"2031-08-16","saturated":false},{"slug":"agieval-en","family":"agieval-en","name":"AGIEval English","nStates":36,"nModels":36,"difficulty":113.9,"saturation":148.7,"saturationExtrapolated":false,"frontierPct":97.6,"extrapolated":false,"extrapolation":null,"maxTested":129.3,"saturationDate":"2025-04-15","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"policybench-us-exact-match","family":"policybench-us-exact-match","name":"PolicyBench US Exact Match","nStates":23,"nModels":23,"difficulty":114.6,"saturation":188.8,"saturationExtrapolated":true,"frontierPct":84.3,"extrapolated":true,"extrapolation":"below","maxTested":161.1,"saturationDate":"2027-12-30","saturationDateLo":"2027-04-27","saturationDateHi":"2028-10-08","saturated":false},{"slug":"teacharena","family":"teacharena","name":"TeachArena","nStates":15,"nModels":15,"difficulty":115.1,"saturation":198.3,"saturationExtrapolated":true,"frontierPct":81.4,"extrapolated":true,"extrapolation":"below","maxTested":161.6,"saturationDate":"2028-06-22","saturationDateLo":"2027-09-23","saturationDateHi":"2029-08-02","saturated":false},{"slug":"arena-hard-v0-1","family":"arena-hard-v0-1","name":"Arena-Hard-Auto v0.1","nStates":74,"nModels":65,"difficulty":115.4,"saturation":135.8,"saturationExtrapolated":false,"frontierPct":99.6,"extrapolated":false,"extrapolation":null,"maxTested":140.8,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2024-12-04","saturated":true},{"slug":"quac","family":"quac","name":"QuAC","nStates":28,"nModels":28,"difficulty":115.4,"saturation":274.2,"saturationExtrapolated":true,"frontierPct":67.8,"extrapolated":false,"extrapolation":null,"maxTested":120.3,"saturationDate":"2033-05-05","saturationDateLo":"2031-08-26","saturationDateHi":"2035-07-21","saturated":false},{"slug":"magi-hard","family":"magi-hard","name":"MAGI-Hard","nStates":92,"nModels":92,"difficulty":115.6,"saturation":141.6,"saturationExtrapolated":false,"frontierPct":99.1,"extrapolated":false,"extrapolation":null,"maxTested":130.0,"saturationDate":"2024-12-17","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"mathvista-mini","family":"mathvista-mini","name":"MathVista (mini)","nStates":135,"nModels":115,"difficulty":116.0,"saturation":166.5,"saturationExtrapolated":false,"frontierPct":93.8,"extrapolated":false,"extrapolation":null,"maxTested":163.8,"saturationDate":"2026-02-11","saturationDateLo":"2025-11-19","saturationDateHi":"2026-02-20","saturated":true},{"slug":"hr-bench-8k","family":"hr-bench-8k","name":"HR-Bench (8K)","nStates":9,"nModels":8,"difficulty":116.1,"saturation":223.8,"saturationExtrapolated":true,"frontierPct":83.8,"extrapolated":false,"extrapolation":null,"maxTested":146.7,"saturationDate":"2029-05-09","saturationDateLo":"2027-05-16","saturationDateHi":"2033-12-15","saturated":false},{"slug":"medscribe","family":"medscribe","name":"MedScribe","nStates":104,"nModels":94,"difficulty":116.2,"saturation":172.1,"saturationExtrapolated":true,"frontierPct":88.3,"extrapolated":true,"extrapolation":"below","maxTested":167.8,"saturationDate":"2027-01-06","saturationDateLo":"2026-09-02","saturationDateHi":"2027-03-14","saturated":false},{"slug":"caselaw-v2","family":"caselaw-v2","name":"CaseLaw","nStates":52,"nModels":52,"difficulty":116.7,"saturation":303.5,"saturationExtrapolated":true,"frontierPct":66.0,"extrapolated":true,"extrapolation":"below","maxTested":161.8,"saturationDate":"2034-10-30","saturationDateLo":"2031-11-25","saturationDateHi":"2040-04-02","saturated":false},{"slug":"mmlu-pro","family":"mmlu-pro","name":"MMLU Pro","nStates":535,"nModels":432,"difficulty":116.7,"saturation":156.6,"saturationExtrapolated":false,"frontierPct":95.1,"extrapolated":false,"extrapolation":null,"maxTested":169.1,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2025-12-11","saturated":true},{"slug":"chinese-simpleqa","family":"chinese-simpleqa","name":"Chinese SimpleQA","nStates":26,"nModels":22,"difficulty":117.3,"saturation":183.2,"saturationExtrapolated":true,"frontierPct":84.8,"extrapolated":false,"extrapolation":null,"maxTested":158.3,"saturationDate":"2027-09-06","saturationDateLo":"2027-02-08","saturationDateHi":"2028-04-16","saturated":false},{"slug":"xbench-scienceqa","family":"xbench-scienceqa","name":"xbench-ScienceQA","nStates":27,"nModels":27,"difficulty":117.5,"saturation":220.6,"saturationExtrapolated":true,"frontierPct":75.6,"extrapolated":true,"extrapolation":"below","maxTested":170.2,"saturationDate":"2030-01-17","saturationDateLo":"2029-02-02","saturationDateHi":"2031-05-22","saturated":false},{"slug":"ocrbench-v2-en","family":"ocrbench-v2","name":"OCRBench v2","nStates":9,"nModels":8,"difficulty":117.9,"saturation":213.2,"saturationExtrapolated":true,"frontierPct":78.4,"extrapolated":false,"extrapolation":null,"maxTested":144.0,"saturationDate":"2029-02-09","saturationDateLo":"2028-01-10","saturationDateHi":"2030-10-06","saturated":false},{"slug":"nextjs-ai-agent-evals","family":"nextjs-ai-agent-evals","name":"Next.js AI Agent Evaluations","nStates":15,"nModels":15,"difficulty":118.2,"saturation":149.2,"saturationExtrapolated":false,"frontierPct":97.8,"extrapolated":true,"extrapolation":"below","maxTested":167.4,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"mmvu","family":"mmvu","name":"MMVU","nStates":48,"nModels":40,"difficulty":118.3,"saturation":183.5,"saturationExtrapolated":true,"frontierPct":84.3,"extrapolated":false,"extrapolation":null,"maxTested":163.7,"saturationDate":"2027-09-14","saturationDateLo":"2027-06-09","saturationDateHi":"2027-12-27","saturated":false},{"slug":"mmmlu","family":"mmmlu","name":"MMMLU","nStates":101,"nModels":75,"difficulty":118.4,"saturation":158.8,"saturationExtrapolated":false,"frontierPct":95.1,"extrapolated":false,"extrapolation":null,"maxTested":163.3,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-03-05","saturated":true},{"slug":"medic-open-ended-elo","family":"medic-open-ended-elo","name":"MEDIC (open-ended Elo)","nStates":49,"nModels":48,"difficulty":118.5,"saturation":167.0,"saturationExtrapolated":false,"frontierPct":90.7,"extrapolated":false,"extrapolation":null,"maxTested":144.4},{"slug":"obviousbench","family":"obviousbench","name":"ObviousBench","nStates":457,"nModels":212,"difficulty":118.5,"saturation":149.6,"saturationExtrapolated":false,"frontierPct":97.7,"extrapolated":false,"extrapolation":null,"maxTested":173.5,"saturationDate":"2025-03-24","saturationDateLo":"2025-03-24","saturationDateHi":"2025-04-15","saturated":true},{"slug":"bigcodebench-complete","family":"bigcodebench-complete","name":"BigCodeBench Complete","nStates":100,"nModels":100,"difficulty":118.8,"saturation":201.0,"saturationExtrapolated":true,"frontierPct":78.7,"extrapolated":false,"extrapolation":null,"maxTested":136.9,"saturationDate":"2028-10-21","saturationDateLo":"2028-02-29","saturationDateHi":"2029-11-03","saturated":false},{"slug":"global-mmlu-lite","family":"global-mmlu-lite","name":"Global MMLU Lite","nStates":118,"nModels":93,"difficulty":118.8,"saturation":159.4,"saturationExtrapolated":false,"frontierPct":95.2,"extrapolated":false,"extrapolation":null,"maxTested":157.1,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"mathverse-mini","family":"mathverse-mini","name":"MathVerse (testmini)","nStates":31,"nModels":24,"difficulty":118.9,"saturation":164.5,"saturationExtrapolated":false,"frontierPct":93.7,"extrapolated":false,"extrapolation":null,"maxTested":152.6,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-11","saturated":true},{"slug":"visual-acuity-4k-threshold-row","family":"visual-acuity-4k-threshold-row","name":"Visual Acuity 4K Threshold Row","nStates":29,"nModels":29,"difficulty":119.9,"saturation":228.2,"saturationExtrapolated":true,"frontierPct":74.9,"extrapolated":false,"extrapolation":null,"maxTested":162.5,"saturationDate":"2030-01-22","saturationDateLo":"2028-04-29","saturationDateHi":"2032-12-08","saturated":false},{"slug":"video-mmmu","family":"video-mmmu","name":"Video-MMMU","nStates":89,"nModels":70,"difficulty":120.0,"saturation":164.1,"saturationExtrapolated":false,"frontierPct":92.4,"extrapolated":false,"extrapolation":null,"maxTested":162.3,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-11","saturationDateHi":"2026-04-08","saturated":true},{"slug":"astabench-2026-scholarqa-cs2","family":"astabench-2026-scholarqa-cs2","name":"ScholarQA-CS2 (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":120.3,"saturation":184.3,"saturationExtrapolated":true,"frontierPct":86.1,"extrapolated":true,"extrapolation":"below","maxTested":162.1,"saturationDate":"2027-06-25","saturationDateLo":"2026-09-02","saturationDateHi":"2029-01-03","saturated":false},{"slug":"astabench-2026-litqa-search","family":"astabench-2026-litqa-search","name":"LitQA2-FullText-Search (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":120.4,"saturation":184.1,"saturationExtrapolated":true,"frontierPct":85.6,"extrapolated":true,"extrapolation":"below","maxTested":162.0,"saturationDate":"2027-07-26","saturationDateLo":"2026-11-14","saturationDateHi":"2028-08-11","saturated":false},{"slug":"mathvision-qwen-corrected","family":"mathvision-qwen-corrected","name":"MathVision - Qwen corrected annotations","nStates":9,"nModels":9,"difficulty":120.4,"saturation":163.4,"saturationExtrapolated":false,"frontierPct":94.6,"extrapolated":true,"extrapolation":"below","maxTested":161.6,"saturationDate":"2026-03-05","saturationDateLo":"2025-12-11","saturationDateHi":"2026-09-02","saturated":true},{"slug":"roboflow-vision-evals","family":"roboflow-vision-evals","name":"Roboflow Vision Evals","nStates":20,"nModels":20,"difficulty":120.5,"saturation":182.7,"saturationExtrapolated":true,"frontierPct":86.6,"extrapolated":true,"extrapolation":"below","maxTested":161.0,"saturationDate":"2027-05-23","saturationDateLo":"2026-09-02","saturationDateHi":"2028-09-16","saturated":false},{"slug":"labbench2","family":"labbench2","name":"LABBench2","nStates":11,"nModels":11,"difficulty":120.7,"saturation":186.2,"saturationExtrapolated":true,"frontierPct":86.3,"extrapolated":true,"extrapolation":"below","maxTested":167.6,"saturationDate":"2027-09-29","saturationDateLo":"2027-01-30","saturationDateHi":"2028-09-18","saturated":false},{"slug":"live-bench","family":"live-bench","name":"LiveBench","nStates":97,"nModels":86,"difficulty":120.7,"saturation":180.2,"saturationExtrapolated":true,"frontierPct":85.9,"extrapolated":false,"extrapolation":null,"maxTested":163.8,"saturationDate":"2027-05-01","saturationDateLo":"2027-01-26","saturationDateHi":"2027-08-24","saturated":false},{"slug":"mmlu-prox","family":"mmlu-prox","name":"MMLU-ProX","nStates":65,"nModels":54,"difficulty":120.8,"saturation":162.9,"saturationExtrapolated":false,"frontierPct":93.0,"extrapolated":false,"extrapolation":null,"maxTested":156.2,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-06-09","saturated":true},{"slug":"dualbench100","family":"dualbench100","name":"DualEntry Accounting AI Benchmark","nStates":22,"nModels":22,"difficulty":120.9,"saturation":181.6,"saturationExtrapolated":true,"frontierPct":87.2,"extrapolated":true,"extrapolation":"below","maxTested":167.2,"saturationDate":"2027-06-04","saturationDateLo":"2026-12-22","saturationDateHi":"2028-02-23","saturated":false},{"slug":"robospatial-home","family":"robospatial-home","name":"RoboSpatial-Home","nStates":11,"nModels":10,"difficulty":121.1,"saturation":240.7,"saturationExtrapolated":true,"frontierPct":73.0,"extrapolated":false,"extrapolation":null,"maxTested":147.5,"saturationDate":"2030-08-28","saturationDateLo":"2028-06-30","saturationDateHi":"2034-05-20","saturated":false},{"slug":"swe-bench-pro-v2","family":"swe-bench-pro-v2","name":"SWE-Bench Pro V2 \u2014 Full","nStates":16,"nModels":16,"difficulty":121.8,"saturation":151.0,"saturationExtrapolated":false,"frontierPct":97.8,"extrapolated":true,"extrapolation":"below","maxTested":168.0,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-12-11","saturated":true},{"slug":"videomme","family":"videomme","name":"Video-MME","nStates":75,"nModels":62,"difficulty":121.8,"saturation":176.7,"saturationExtrapolated":true,"frontierPct":90.3,"extrapolated":false,"extrapolation":null,"maxTested":152.7,"saturationDate":"2027-02-22","saturationDateLo":"2026-11-07","saturationDateHi":"2027-05-30","saturated":false},{"slug":"mmstar","family":"mmstar-vlmevalkit","name":"MMStar (OpenVLM)","nStates":96,"nModels":82,"difficulty":121.9,"saturation":186.1,"saturationExtrapolated":true,"frontierPct":89.2,"extrapolated":false,"extrapolation":null,"maxTested":161.6,"saturationDate":"2027-06-07","saturationDateLo":"2027-03-11","saturationDateHi":"2027-09-06","saturated":false},{"slug":"smt-2025-caisi-58","family":"smt-2025-caisi-58","name":"SMT 2025 (CAISI 58-task set)","nStates":13,"nModels":13,"difficulty":122.4,"saturation":147.9,"saturationExtrapolated":false,"frontierPct":98.3,"extrapolated":true,"extrapolation":"below","maxTested":163.8,"saturationDate":"2025-04-15","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"mmmu","family":"mmmu","name":"MMMU","nStates":164,"nModels":134,"difficulty":122.5,"saturation":169.4,"saturationExtrapolated":false,"frontierPct":92.3,"extrapolated":false,"extrapolation":null,"maxTested":158.3,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2026-09-02","saturated":true},{"slug":"perception-test-mcqa","family":"perception-test-mcqa","name":"Perception Test (MCQA)","nStates":12,"nModels":12,"difficulty":122.9,"saturation":215.7,"saturationExtrapolated":true,"frontierPct":84.1,"extrapolated":false,"extrapolation":null,"maxTested":144.9,"saturationDate":"2029-05-15","saturationDateLo":"2027-10-24","saturationDateHi":"2031-08-25","saturated":false},{"slug":"bedside-bench","family":"bedside-bench","name":"Bedside Bench","nStates":15,"nModels":15,"difficulty":123.1,"saturation":165.4,"saturationExtrapolated":false,"frontierPct":93.7,"extrapolated":true,"extrapolation":"below","maxTested":168.6,"saturationDate":"2026-04-22","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-02","saturated":true},{"slug":"biomysterybench-human-solvable","family":"biomysterybench-human-solvable","name":"BioMysteryBench - Human Solvable","nStates":9,"nModels":9,"difficulty":123.1,"saturation":169.0,"saturationExtrapolated":true,"frontierPct":92.4,"extrapolated":true,"extrapolation":"below","maxTested":165.2,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2027-01-28","saturated":true},{"slug":"point-bench","family":"point-bench","name":"Point-Bench","nStates":9,"nModels":8,"difficulty":123.1,"saturation":221.5,"saturationExtrapolated":true,"frontierPct":77.6,"extrapolated":false,"extrapolation":null,"maxTested":151.1,"saturationDate":"2029-02-05","saturationDateLo":"2027-01-15","saturationDateHi":"2033-11-22","saturated":false},{"slug":"design-creativity-bench-appropriateness","family":"design-creativity-bench","name":"Design Creativity Bench","nStates":14,"nModels":14,"difficulty":123.3,"saturation":147.1,"saturationExtrapolated":false,"frontierPct":98.8,"extrapolated":true,"extrapolation":"below","maxTested":169.9,"saturationDate":"2025-04-15","saturationDateLo":"2024-12-04","saturationDateHi":"2025-07-31","saturated":true},{"slug":"mmlu-pro-openllm-harness","family":"mmlu-pro-openllm-harness","name":"MMLU-Pro (Open LLM Leaderboard v2)","nStates":121,"nModels":121,"difficulty":123.6,"saturation":163.4,"saturationExtrapolated":false,"frontierPct":93.1,"extrapolated":false,"extrapolation":null,"maxTested":126.1,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-01","saturated":true},{"slug":"collie","family":"collie","name":"COLLIE","nStates":13,"nModels":13,"difficulty":123.8,"saturation":147.1,"saturationExtrapolated":false,"frontierPct":99.6,"extrapolated":false,"extrapolation":null,"maxTested":156.2,"saturationDate":"2024-09-12","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"multipl-e-humaneval-bash","family":"multipl-e-humaneval-bash","name":"MultiPL-E HumanEval (Bash)","nStates":18,"nModels":18,"difficulty":124.0,"saturation":165.4,"saturationExtrapolated":false,"frontierPct":92.0,"extrapolated":false,"extrapolation":null,"maxTested":125.8,"saturationDate":"2026-04-07","saturationDateLo":"2025-07-31","saturationDateHi":"2027-08-17","saturated":true},{"slug":"nexus","family":"nexus","name":"Nexus Function Calling","nStates":13,"nModels":13,"difficulty":124.2,"saturation":176.2,"saturationExtrapolated":true,"frontierPct":87.5,"extrapolated":false,"extrapolation":null,"maxTested":130.0,"saturationDate":"2026-12-20","saturationDateLo":"2026-02-11","saturationDateHi":"2028-04-21","saturated":false},{"slug":"sab-research","family":"sab-research","name":"SAB Research","nStates":13,"nModels":13,"difficulty":124.3,"saturation":155.7,"saturationExtrapolated":false,"frontierPct":96.2,"extrapolated":true,"extrapolation":"below","maxTested":168.1,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-04","saturated":true},{"slug":"vibe-eval-gemini-judge","family":"vibe-eval-gemini-judge","name":"Vibe-Eval (Gemini judge)","nStates":16,"nModels":12,"difficulty":124.4,"saturation":188.4,"saturationExtrapolated":true,"frontierPct":82.0,"extrapolated":false,"extrapolation":null,"maxTested":145.9,"saturationDate":"2028-01-04","saturationDateLo":"2027-07-15","saturationDateHi":"2028-08-22","saturated":false},{"slug":"astabench-2026-ds1000","family":"astabench-2026-ds1000","name":"DS-1000 (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":124.5,"saturation":173.3,"saturationExtrapolated":true,"frontierPct":88.3,"extrapolated":true,"extrapolation":"below","maxTested":161.8,"saturationDate":"2026-12-03","saturationDateLo":"2026-09-02","saturationDateHi":"2027-05-08","saturated":false},{"slug":"brumo-2025-matharena","family":"brumo-2025-matharena","name":"BRUMO 2025","nStates":45,"nModels":42,"difficulty":124.7,"saturation":146.3,"saturationExtrapolated":false,"frontierPct":99.3,"extrapolated":true,"extrapolation":"below","maxTested":158.4,"saturationDate":"2025-03-24","saturationDateLo":"2024-09-12","saturationDateHi":"2025-03-24","saturated":true},{"slug":"mortgage_tax-vals-all-1258","family":"mortgage_tax-vals-all-1258","name":"MortgageTax (Vals all 1,258 documents)","nStates":94,"nModels":85,"difficulty":124.7,"saturation":234.1,"saturationExtrapolated":true,"frontierPct":71.9,"extrapolated":false,"extrapolation":null,"maxTested":168.0,"saturationDate":"2030-09-09","saturationDateLo":"2029-11-29","saturationDateHi":"2031-11-20","saturated":false},{"slug":"agentic-tool-use-enterprise-scale-ai-2","family":"agentic-tool-use-enterprise-scale-ai-2","name":"Agentic Tool Use (Enterprise)","nStates":33,"nModels":31,"difficulty":124.8,"saturation":167.0,"saturationExtrapolated":false,"frontierPct":90.6,"extrapolated":false,"extrapolation":null,"maxTested":143.9,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2026-12-19","saturated":true},{"slug":"smt-2025-matharena","family":"smt-2025-matharena","name":"SMT 2025","nStates":43,"nModels":40,"difficulty":125.2,"saturation":154.1,"saturationExtrapolated":false,"frontierPct":97.5,"extrapolated":true,"extrapolation":"below","maxTested":158.1,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"medmcqa","family":"medmcqa","name":"MedMCQA","nStates":71,"nModels":58,"difficulty":125.6,"saturation":182.9,"saturationExtrapolated":true,"frontierPct":87.9,"extrapolated":false,"extrapolation":null,"maxTested":155.7,"saturationDate":"2027-08-18","saturationDateLo":"2027-03-31","saturationDateHi":"2028-01-13","saturated":false},{"slug":"logicvista","family":"logicvista","name":"LogicVista","nStates":32,"nModels":26,"difficulty":125.7,"saturation":176.9,"saturationExtrapolated":true,"frontierPct":87.7,"extrapolated":false,"extrapolation":null,"maxTested":151.4,"saturationDate":"2026-12-26","saturationDateLo":"2026-09-02","saturationDateHi":"2027-04-22","saturated":false},{"slug":"roboflow-vision-evals-2026-09","family":"roboflow-vision-evals-2026-09","name":"Roboflow Vision Evals (September 2026 protocol)","nStates":41,"nModels":41,"difficulty":125.7,"saturation":192.3,"saturationExtrapolated":true,"frontierPct":81.7,"extrapolated":true,"extrapolation":"below","maxTested":167.2,"saturationDate":"2028-02-27","saturationDateLo":"2027-07-06","saturationDateHi":"2029-01-28","saturated":false},{"slug":"epiq-corporatebench","family":"epiq-corporatebench","name":"Epiq LAER CorporateBench","nStates":10,"nModels":10,"difficulty":126.0,"saturation":198.6,"saturationExtrapolated":true,"frontierPct":80.8,"extrapolated":true,"extrapolation":"below","maxTested":169.2,"saturationDate":"2028-07-11","saturationDateLo":"2027-09-16","saturationDateHi":"2029-10-29","saturated":false},{"slug":"coding-scale-ai-2","family":"coding-scale-ai-2","name":"Coding","nStates":28,"nModels":28,"difficulty":126.2,"saturation":152.2,"saturationExtrapolated":false,"frontierPct":97.4,"extrapolated":false,"extrapolation":null,"maxTested":140.8},{"slug":"alpacaeval-2-lc-winrate","family":"alpacaeval-2-lc-winrate","name":"AlpacaEval 2.0 (length-controlled win rate)","nStates":61,"nModels":61,"difficulty":126.4,"saturation":164.2,"saturationExtrapolated":false,"frontierPct":91.9,"extrapolated":false,"extrapolation":null,"maxTested":137.1,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-02","saturated":true},{"slug":"hmmt-nov-2025-matharena","family":"hmmt-nov-2025-matharena","name":"HMMT Nov 2025","nStates":51,"nModels":45,"difficulty":126.5,"saturation":147.4,"saturationExtrapolated":false,"frontierPct":98.8,"extrapolated":false,"extrapolation":null,"maxTested":154.9,"saturationDate":"2025-04-15","saturationDateLo":"2025-03-24","saturationDateHi":"2025-06-09","saturated":true},{"slug":"buzzbench","family":"buzzbench","name":"BuzzBench","nStates":21,"nModels":21,"difficulty":126.6,"saturation":176.0,"saturationExtrapolated":true,"frontierPct":86.4,"extrapolated":false,"extrapolation":null,"maxTested":143.8,"saturationDate":"2027-03-15","saturationDateLo":"2026-09-02","saturationDateHi":"2027-08-23","saturated":false},{"slug":"include","family":"include","name":"INCLUDE","nStates":27,"nModels":27,"difficulty":126.7,"saturation":165.9,"saturationExtrapolated":false,"frontierPct":93.3,"extrapolated":false,"extrapolation":null,"maxTested":156.1,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-02","saturated":true},{"slug":"aime-2026-matharena","family":"aime-2026-matharena","name":"AIME 2026","nStates":72,"nModels":58,"difficulty":127.1,"saturation":146.7,"saturationExtrapolated":false,"frontierPct":99.3,"extrapolated":false,"extrapolation":null,"maxTested":165.2,"saturationDate":"2025-03-24","saturationDateLo":"2025-03-24","saturationDateHi":"2025-03-24","saturated":true},{"slug":"japanese-scale-ai-2","family":"japanese-scale-ai-2","name":"Japanese","nStates":17,"nModels":17,"difficulty":127.4,"saturation":159.2,"saturationExtrapolated":false,"frontierPct":95.3,"extrapolated":false,"extrapolation":null,"maxTested":140.8},{"slug":"we-math-testmini-strict","family":"we-math-testmini-strict","name":"We-Math (testmini, strict score)","nStates":9,"nModels":8,"difficulty":127.6,"saturation":157.5,"saturationExtrapolated":false,"frontierPct":96.7,"extrapolated":false,"extrapolation":null,"maxTested":145.3,"saturationDate":"2025-07-31","saturationDateLo":"2025-04-15","saturationDateHi":"2025-12-11","saturated":true},{"slug":"spanish-scale-ai-2","family":"spanish-scale-ai-2","name":"Spanish","nStates":22,"nModels":22,"difficulty":127.9,"saturation":155.2,"saturationExtrapolated":false,"frontierPct":97.0,"extrapolated":false,"extrapolation":null,"maxTested":140.8},{"slug":"arabic-scale-ai-2","family":"arabic-scale-ai-2","name":"Arabic","nStates":18,"nModels":18,"difficulty":128.1,"saturation":175.6,"saturationExtrapolated":true,"frontierPct":89.5,"extrapolated":false,"extrapolation":null,"maxTested":140.8},{"slug":"charxiv-reasoning","family":"charxiv-reasoning","name":"CharXiv Reasoning","nStates":168,"nModels":129,"difficulty":128.2,"saturation":174.5,"saturationExtrapolated":true,"frontierPct":87.6,"extrapolated":false,"extrapolation":null,"maxTested":166.2,"saturationDate":"2027-01-01","saturationDateLo":"2026-09-02","saturationDateHi":"2027-02-26","saturated":false},{"slug":"chinese-scale-ai-2","family":"chinese-scale-ai-2","name":"Chinese","nStates":21,"nModels":21,"difficulty":128.2,"saturation":156.9,"saturationExtrapolated":false,"frontierPct":96.3,"extrapolated":false,"extrapolation":null,"maxTested":140.8},{"slug":"nimble-web-context","family":"nimble-web-context","name":"Nimble Web Context","nStates":11,"nModels":11,"difficulty":128.3,"saturation":206.9,"saturationExtrapolated":true,"frontierPct":78.7,"extrapolated":true,"extrapolation":"below","maxTested":168.7,"saturationDate":"2028-11-30","saturationDateLo":"2027-11-17","saturationDateHi":"2030-11-30","saturated":false},{"slug":"llm-persuasion-average-strength","family":"llm-persuasion-average-strength","name":"LLM Persuasion Average Strength","nStates":15,"nModels":15,"difficulty":128.5,"saturation":248.4,"saturationExtrapolated":true,"frontierPct":68.2,"extrapolated":true,"extrapolation":"below","maxTested":157.7,"saturationDate":"2031-07-16","saturationDateLo":"2030-03-30","saturationDateHi":"2033-11-16","saturated":false},{"slug":"mmmu-pro-vals-4-option","family":"mmmu-pro-vals-4-option","name":"MMMU-Pro (Vals 4-Option)","nStates":92,"nModels":83,"difficulty":128.6,"saturation":176.5,"saturationExtrapolated":true,"frontierPct":89.7,"extrapolated":false,"extrapolation":null,"maxTested":167.1,"saturationDate":"2027-03-07","saturationDateLo":"2026-12-08","saturationDateHi":"2027-06-14","saturated":false},{"slug":"tau2-bench-average","family":"tau2-bench-average","name":"\u03c4\u00b2-bench (domain average)","nStates":21,"nModels":21,"difficulty":128.7,"saturation":159.8,"saturationExtrapolated":false,"frontierPct":94.3,"extrapolated":false,"extrapolation":null,"maxTested":152.7,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-04-07","saturated":true},{"slug":"vending-bench-days-until-sales-stop-percentage","family":"vending-bench","name":"Vending-Bench","nStates":21,"nModels":21,"difficulty":128.7,"saturation":152.7,"saturationExtrapolated":false,"frontierPct":98.4,"extrapolated":false,"extrapolation":null,"maxTested":152.7,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2026-02-11","saturated":true},{"slug":"snorkel-wordle","family":"snorkel-wordle","name":"SnorkelWordle","nStates":20,"nModels":20,"difficulty":128.8,"saturation":151.3,"saturationExtrapolated":false,"frontierPct":98.4,"extrapolated":false,"extrapolation":null,"maxTested":154.2,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-12-11","saturated":true},{"slug":"livecodebench-vals-public","family":"livecodebench-vals-public","name":"LiveCodeBench (Vals AI public dataset)","nStates":134,"nModels":122,"difficulty":129.5,"saturation":162.4,"saturationExtrapolated":false,"frontierPct":93.7,"extrapolated":false,"extrapolation":null,"maxTested":167.7,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-04","saturationDateHi":"2026-02-11","saturated":true},{"slug":"tau-bench-retail","family":"tau-bench-retail","name":"\u03c4-bench (Retail)","nStates":57,"nModels":40,"difficulty":129.5,"saturation":160.4,"saturationExtrapolated":false,"frontierPct":93.8,"extrapolated":false,"extrapolation":null,"maxTested":147.9,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-11","saturationDateHi":"2026-04-22","saturated":true},{"slug":"judgemark-v2-1","family":"judgemark-v2-1","name":"Judgemark v2.1","nStates":46,"nModels":45,"difficulty":129.6,"saturation":160.2,"saturationExtrapolated":false,"frontierPct":93.9,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-11","saturationDateHi":"2026-04-07","saturated":true},{"slug":"we-math","family":"we-math","name":"We-Math","nStates":10,"nModels":10,"difficulty":129.7,"saturation":161.1,"saturationExtrapolated":false,"frontierPct":95.0,"extrapolated":true,"extrapolation":"below","maxTested":152.7,"saturationDate":"2026-02-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-09-02","saturated":true},{"slug":"snorkel-sequences","family":"snorkel-sequences","name":"SnorkelSequences","nStates":31,"nModels":31,"difficulty":130.0,"saturation":191.9,"saturationExtrapolated":true,"frontierPct":80.6,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2028-01-05","saturationDateLo":"2027-04-05","saturationDateHi":"2029-04-30","saturated":false},{"slug":"tau2-bench-retail","family":"tau2-bench-retail","name":"\u03c4\u00b2-bench (Retail)","nStates":89,"nModels":71,"difficulty":130.0,"saturation":164.3,"saturationExtrapolated":false,"frontierPct":91.6,"extrapolated":false,"extrapolation":null,"maxTested":159.5,"saturationDate":"2026-06-09","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-02","saturated":true},{"slug":"agentic-tool-use-chat-scale-ai-2","family":"agentic-tool-use-chat-scale-ai-2","name":"Agentic Tool Use (Chat)","nStates":32,"nModels":30,"difficulty":130.1,"saturation":172.8,"saturationExtrapolated":true,"frontierPct":88.0,"extrapolated":false,"extrapolation":null,"maxTested":144.3,"saturationDate":"2026-12-09","saturationDateLo":"2026-09-01","saturationDateHi":"2027-07-17","saturated":false},{"slug":"psebench","family":"psebench","name":"PSEBench","nStates":12,"nModels":12,"difficulty":130.3,"saturation":159.2,"saturationExtrapolated":false,"frontierPct":96.7,"extrapolated":false,"extrapolation":null,"maxTested":159.0,"saturationDate":"2026-02-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-06-09","saturated":true},{"slug":"taxbench-total-pass1","family":"taxbench-total-pass1","name":"TaxBench Total pass@1","nStates":15,"nModels":14,"difficulty":130.4,"saturation":205.5,"saturationExtrapolated":true,"frontierPct":77.2,"extrapolated":true,"extrapolation":"below","maxTested":162.9,"saturationDate":"2028-12-07","saturationDateLo":"2027-11-28","saturationDateHi":"2030-08-17","saturated":false},{"slug":"corp_fin_v2","family":"corp_fin_v2","name":"CorpFin v2","nStates":131,"nModels":121,"difficulty":130.6,"saturation":214.1,"saturationExtrapolated":true,"frontierPct":72.5,"extrapolated":false,"extrapolation":null,"maxTested":165.0,"saturationDate":"2029-09-23","saturationDateLo":"2029-05-08","saturationDateHi":"2030-02-19","saturated":false},{"slug":"math-lvl-5-openllm-harness","family":"math-lvl-5-openllm-harness","name":"MATH Level 5 (Open LLM Leaderboard v2)","nStates":121,"nModels":121,"difficulty":130.6,"saturation":160.4,"saturationExtrapolated":true,"frontierPct":94.3,"extrapolated":true,"extrapolation":"above","maxTested":126.1,"saturationDate":"2026-02-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-07-24","saturated":true},{"slug":"snorkel-underwrite","family":"snorkel-underwrite","name":"SnorkelUnderwrite","nStates":32,"nModels":32,"difficulty":130.6,"saturation":170.1,"saturationExtrapolated":true,"frontierPct":89.5,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2026-09-02","saturationDateLo":"2026-04-07","saturationDateHi":"2027-04-14","saturated":true},{"slug":"charxiv-reasoning-with-tools-card","family":"charxiv-reasoning-with-tools-card","name":"CharXiv Reasoning (with tools)","nStates":53,"nModels":41,"difficulty":131.3,"saturation":161.2,"saturationExtrapolated":false,"frontierPct":93.9,"extrapolated":true,"extrapolation":"below","maxTested":165.8,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-11","saturationDateHi":"2026-03-05","saturated":true},{"slug":"lingoqa","family":"lingoqa","name":"LingoQA","nStates":21,"nModels":21,"difficulty":131.3,"saturation":188.0,"saturationExtrapolated":true,"frontierPct":84.1,"extrapolated":true,"extrapolation":"below","maxTested":161.6,"saturationDate":"2027-08-10","saturationDateLo":"2026-09-02","saturationDateHi":"2029-03-29","saturated":false},{"slug":"imo-answerbench","family":"imo-answerbench","name":"IMO-AnswerBench","nStates":47,"nModels":35,"difficulty":131.5,"saturation":161.1,"saturationExtrapolated":false,"frontierPct":95.4,"extrapolated":false,"extrapolation":null,"maxTested":163.5,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-11","saturated":true},{"slug":"aime-2024","family":"aime-2024","name":"AIME 2024","nStates":133,"nModels":106,"difficulty":131.6,"saturation":155.7,"saturationExtrapolated":false,"frontierPct":98.2,"extrapolated":false,"extrapolation":null,"maxTested":155.9,"saturationDate":"2025-03-24","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"aime-2025","family":"aime-2025","name":"AIME 2025","nStates":224,"nModels":185,"difficulty":131.6,"saturation":152.6,"saturationExtrapolated":false,"frontierPct":98.8,"extrapolated":false,"extrapolation":null,"maxTested":176.4,"saturationDate":"2025-03-24","saturationDateLo":"2025-03-24","saturationDateHi":"2025-06-05","saturated":true},{"slug":"livecodebench-v6-reported","family":"livecodebench","name":"LiveCodeBench","nStates":92,"nModels":75,"difficulty":131.7,"saturation":158.7,"saturationExtrapolated":false,"frontierPct":95.6,"extrapolated":false,"extrapolation":null,"maxTested":159.5,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-11","saturated":true},{"slug":"nova-63","family":"nova-63","name":"NOVA-63","nStates":27,"nModels":27,"difficulty":131.8,"saturation":294.1,"saturationExtrapolated":true,"frontierPct":62.1,"extrapolated":false,"extrapolation":null,"maxTested":156.1,"saturationDate":"2034-11-26","saturationDateLo":"2033-07-07","saturationDateHi":"2036-07-13","saturated":false},{"slug":"gorilla-apibench","family":"gorilla-apibench","name":"Gorilla APIBench","nStates":9,"nModels":9,"difficulty":132.1,"saturation":155.5,"saturationExtrapolated":true,"frontierPct":97.4,"extrapolated":true,"extrapolation":"above","maxTested":130.0,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2026-11-16","saturated":true},{"slug":"natural-questions","family":"natural-questions","name":"Natural Questions (open-domain, closed-book)","nStates":57,"nModels":57,"difficulty":132.3,"saturation":214.4,"saturationExtrapolated":true,"frontierPct":72.6,"extrapolated":true,"extrapolation":"above","maxTested":123.9,"saturationDate":"2029-07-24","saturationDateLo":"2028-09-17","saturationDateHi":"2030-09-07","saturated":false},{"slug":"mmmu-pro","family":"mmmu-pro","name":"MMMU Pro","nStates":339,"nModels":189,"difficulty":132.4,"saturation":178.1,"saturationExtrapolated":true,"frontierPct":86.6,"extrapolated":false,"extrapolation":null,"maxTested":169.5,"saturationDate":"2027-04-28","saturationDateLo":"2027-02-18","saturationDateHi":"2027-05-31","saturated":false},{"slug":"zeroeval-zebralogic","family":"zeroeval-zebralogic","name":"ZeroEval ZebraLogic","nStates":55,"nModels":52,"difficulty":132.4,"saturation":154.9,"saturationExtrapolated":false,"frontierPct":98.1,"extrapolated":false,"extrapolation":null,"maxTested":144.8,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"global-piqa-nonparallel-exact-match-strict","family":"global-piqa-nonparallel-exact-match-strict","name":"Global PIQA - Non-Parallel (Strict Exact Match)","nStates":54,"nModels":54,"difficulty":132.5,"saturation":179.2,"saturationExtrapolated":true,"frontierPct":92.3,"extrapolated":false,"extrapolation":null,"maxTested":149.0,"saturationDate":"2027-05-02","saturationDateLo":"2026-09-02","saturationDateHi":"2028-03-27","saturated":false},{"slug":"graphwalks-bfs-up-to-128k","family":"graphwalks-bfs-up-to-128k","name":"GraphWalks BFS: Up to 128K","nStates":22,"nModels":22,"difficulty":132.5,"saturation":159.1,"saturationExtrapolated":false,"frontierPct":95.1,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-03-05","saturated":true},{"slug":"advancedif","family":"advancedif","name":"AdvancedIF","nStates":42,"nModels":40,"difficulty":132.7,"saturation":185.0,"saturationExtrapolated":true,"frontierPct":83.0,"extrapolated":true,"extrapolation":"below","maxTested":170.5,"saturationDate":"2027-08-13","saturationDateLo":"2027-04-13","saturationDateHi":"2028-03-03","saturated":false},{"slug":"cyberbench-overall","family":"cyberbench-overall","name":"CyberBench Overall","nStates":19,"nModels":19,"difficulty":132.8,"saturation":215.2,"saturationExtrapolated":true,"frontierPct":76.5,"extrapolated":true,"extrapolation":"below","maxTested":165.3,"saturationDate":"2029-05-20","saturationDateLo":"2027-10-30","saturationDateHi":"2032-08-17","saturated":false},{"slug":"livecodebench-artificial-analysis","family":"livecodebench-artificial-analysis","name":"LiveCodeBench (Artificial Analysis)","nStates":330,"nModels":265,"difficulty":132.9,"saturation":160.8,"saturationExtrapolated":false,"frontierPct":95.0,"extrapolated":false,"extrapolation":null,"maxTested":159.0,"saturationDate":"2026-02-04","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"gpqa-diamond","family":"gpqa-diamond","name":"GPQA Diamond","nStates":563,"nModels":356,"difficulty":133.1,"saturation":159.7,"saturationExtrapolated":false,"frontierPct":96.3,"extrapolated":false,"extrapolation":null,"maxTested":170.1,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"slop-score","family":"slop-score","name":"Slop Score","nStates":19,"nModels":19,"difficulty":133.1,"saturation":194.5,"saturationExtrapolated":true,"frontierPct":80.4,"extrapolated":false,"extrapolation":null,"maxTested":147.2,"saturationDate":"2027-12-30","saturationDateLo":"2026-12-29","saturationDateHi":"2030-03-27","saturated":false},{"slug":"snorkel-finance","family":"snorkel-finance","name":"SnorkelFinance","nStates":27,"nModels":27,"difficulty":133.1,"saturation":158.9,"saturationExtrapolated":false,"frontierPct":95.0,"extrapolated":false,"extrapolation":null,"maxTested":153.4,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-04-07","saturated":true},{"slug":"theoremqa","family":"theoremqa","name":"TheoremQA","nStates":47,"nModels":47,"difficulty":133.2,"saturation":199.5,"saturationExtrapolated":true,"frontierPct":76.4,"extrapolated":true,"extrapolation":"above","maxTested":122.3,"saturationDate":"2028-08-26","saturationDateLo":"2027-11-26","saturationDateHi":"2029-05-02","saturated":false},{"slug":"visual-math-overall-matharena","family":"visual-math-overall-matharena","name":"Visual Math \u2014 Overall","nStates":22,"nModels":22,"difficulty":133.2,"saturation":167.4,"saturationExtrapolated":false,"frontierPct":92.9,"extrapolated":true,"extrapolation":"below","maxTested":161.5,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2026-12-25","saturated":true},{"slug":"simplevqa","family":"simplevqa","name":"SimpleVQA","nStates":35,"nModels":32,"difficulty":133.6,"saturation":207.3,"saturationExtrapolated":true,"frontierPct":73.9,"extrapolated":false,"extrapolation":null,"maxTested":161.6,"saturationDate":"2029-03-28","saturationDateLo":"2028-09-06","saturationDateHi":"2029-11-15","saturated":false},{"slug":"tau2-bench-telecom","family":"tau2-bench-telecom","name":"\u03c4\u00b2-bench (Telecom)","nStates":460,"nModels":325,"difficulty":133.8,"saturation":167.2,"saturationExtrapolated":false,"frontierPct":90.4,"extrapolated":false,"extrapolation":null,"maxTested":164.1,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2026-11-02","saturated":true},{"slug":"androidworld-reported","family":"androidworld-reported","name":"AndroidWorld - release-reported mobile use","nStates":33,"nModels":33,"difficulty":134.1,"saturation":178.7,"saturationExtrapolated":true,"frontierPct":85.0,"extrapolated":false,"extrapolation":null,"maxTested":161.6,"saturationDate":"2027-03-27","saturationDateLo":"2026-11-09","saturationDateHi":"2027-08-29","saturated":false},{"slug":"pumac-2024-caisi-55","family":"pumac-2024-caisi-55","name":"PUMaC 2024 (CAISI 55-task set)","nStates":13,"nModels":13,"difficulty":134.1,"saturation":151.3,"saturationExtrapolated":false,"frontierPct":98.8,"extrapolated":true,"extrapolation":"below","maxTested":163.8,"saturationDate":"2025-07-31","saturationDateLo":"2025-07-31","saturationDateHi":"2025-07-31","saturated":true},{"slug":"math-vision","family":"math-vision","name":"MATH-Vision","nStates":120,"nModels":100,"difficulty":134.2,"saturation":161.7,"saturationExtrapolated":false,"frontierPct":95.0,"extrapolated":false,"extrapolation":null,"maxTested":167.1,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"hmmt-feb-2026-matharena","family":"hmmt-feb-2026-matharena","name":"HMMT Feb 2026","nStates":53,"nModels":44,"difficulty":134.3,"saturation":153.8,"saturationExtrapolated":false,"frontierPct":98.4,"extrapolated":false,"extrapolation":null,"maxTested":165.4,"saturationDate":"2025-07-31","saturationDateLo":"2025-07-31","saturationDateHi":"2025-07-31","saturated":true},{"slug":"pgg-bench-trueskill-mu","family":"pgg-bench-trueskill-mu","name":"PGG-Bench TrueSkill Mu","nStates":18,"nModels":17,"difficulty":134.3,"saturation":197.4,"saturationExtrapolated":true,"frontierPct":83.2,"extrapolated":false,"extrapolation":null,"maxTested":144.9},{"slug":"otis-mock-aime-2024-2025","family":"otis-mock-aime-2024-2025","name":"OTIS Mock AIME 2024-2025","nStates":211,"nModels":131,"difficulty":134.5,"saturation":155.0,"saturationExtrapolated":false,"frontierPct":97.9,"extrapolated":false,"extrapolation":null,"maxTested":172.1,"saturationDate":"2025-07-31","saturationDateLo":"2025-07-31","saturationDateHi":"2025-07-31","saturated":true},{"slug":"bazaar-trueskill-mu","family":"bazaar-trueskill-mu","name":"BAZAAR TrueSkill Mu","nStates":24,"nModels":22,"difficulty":134.7,"saturation":159.5,"saturationExtrapolated":false,"frontierPct":95.2,"extrapolated":false,"extrapolation":null,"maxTested":147.4},{"slug":"snakebench-rating","family":"snakebench-rating","name":"SnakeBench Rating","nStates":208,"nModels":205,"difficulty":134.8,"saturation":188.9,"saturationExtrapolated":true,"frontierPct":79.7,"extrapolated":false,"extrapolation":null,"maxTested":165.5},{"slug":"arena-hard-v2","family":"arena-hard-v2","name":"Arena-Hard-Auto v2.0","nStates":48,"nModels":40,"difficulty":135.1,"saturation":155.8,"saturationExtrapolated":false,"frontierPct":97.2,"extrapolated":false,"extrapolation":null,"maxTested":154.1,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-04","saturated":true},{"slug":"aikido-known-cve-detection","family":"aikido-known-cve-detection","name":"Aikido Known-CVE Detection (Pass@3 Union)","nStates":23,"nModels":13,"difficulty":135.4,"saturation":187.0,"saturationExtrapolated":true,"frontierPct":83.3,"extrapolated":true,"extrapolation":"below","maxTested":165.4,"saturationDate":"2027-07-27","saturationDateLo":"2026-12-23","saturationDateHi":"2028-10-08","saturated":false},{"slug":"otis-aime-2025-caisi-30","family":"otis-aime-2025-caisi-30","name":"OTIS-AIME 2025 (CAISI 30-task set)","nStates":13,"nModels":13,"difficulty":135.4,"saturation":150.2,"saturationExtrapolated":false,"frontierPct":99.3,"extrapolated":true,"extrapolation":"below","maxTested":164.1,"saturationDate":"2025-07-31","saturationDateLo":"2025-04-15","saturationDateHi":"2025-07-31","saturated":true},{"slug":"snorkel-graph","family":"snorkel-graph","name":"SnorkelGraph","nStates":30,"nModels":30,"difficulty":135.4,"saturation":166.0,"saturationExtrapolated":false,"frontierPct":91.6,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2026-06-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-11-13","saturated":true},{"slug":"global-piqa-parallel-exact-match-strict","family":"global-piqa-parallel-exact-match-strict","name":"Global PIQA - Parallel (Strict Exact Match)","nStates":53,"nModels":53,"difficulty":135.6,"saturation":174.1,"saturationExtrapolated":true,"frontierPct":90.1,"extrapolated":false,"extrapolation":null,"maxTested":149.0,"saturationDate":"2027-01-08","saturationDateLo":"2026-09-01","saturationDateHi":"2027-07-08","saturated":false},{"slug":"veracode-genai-code-security","family":"veracode-genai-code-security","name":"Veracode GenAI Code Security","nStates":9,"nModels":9,"difficulty":135.7,"saturation":211.8,"saturationExtrapolated":true,"frontierPct":76.8,"extrapolated":true,"extrapolation":"below","maxTested":158.6,"saturationDate":"2029-02-08","saturationDateLo":"2027-06-11","saturationDateHi":"2032-04-14","saturated":false},{"slug":"aml-bench","family":"aml-bench","name":"AML Bench","nStates":11,"nModels":11,"difficulty":135.8,"saturation":228.4,"saturationExtrapolated":true,"frontierPct":71.4,"extrapolated":true,"extrapolation":"below","maxTested":167.2,"saturationDate":"2030-03-15","saturationDateLo":"2028-08-21","saturationDateHi":"2032-11-30","saturated":false},{"slug":"bfcl-v4-overall-accuracy","family":"bfcl-v4-overall-accuracy","name":"BFCL v4 Overall Accuracy","nStates":92,"nModels":85,"difficulty":136.0,"saturation":190.9,"saturationExtrapolated":true,"frontierPct":78.4,"extrapolated":false,"extrapolation":null,"maxTested":160.8,"saturationDate":"2028-03-08","saturationDateLo":"2027-10-17","saturationDateHi":"2028-08-11","saturated":false},{"slug":"lmarena-text-arena-elo","family":"lmarena-text-arena-elo","name":"LMArena Text Arena Elo","nStates":258,"nModels":240,"difficulty":136.0,"saturation":184.2,"saturationExtrapolated":true,"frontierPct":80.6,"extrapolated":false,"extrapolation":null,"maxTested":167.7},{"slug":"hmmt-feb-2025-matharena","family":"hmmt-feb-2025-matharena","name":"HMMT Feb 2025","nStates":95,"nModels":78,"difficulty":136.1,"saturation":152.8,"saturationExtrapolated":false,"frontierPct":99.2,"extrapolated":false,"extrapolation":null,"maxTested":161.8,"saturationDate":"2025-07-31","saturationDateLo":"2025-03-24","saturationDateHi":"2025-07-31","saturated":true},{"slug":"dtbench","family":"dtbench","name":"DTBench","nStates":230,"nModels":174,"difficulty":136.2,"saturation":162.2,"saturationExtrapolated":false,"frontierPct":96.4,"extrapolated":false,"extrapolation":null,"maxTested":170.2,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-11","saturationDateHi":"2026-03-05","saturated":true},{"slug":"cmimc-2025-matharena","family":"cmimc-2025-matharena","name":"CMIMC 2025","nStates":36,"nModels":33,"difficulty":136.5,"saturation":156.2,"saturationExtrapolated":false,"frontierPct":98.6,"extrapolated":false,"extrapolation":null,"maxTested":159.2,"saturationDate":"2025-07-31","saturationDateLo":"2025-04-15","saturationDateHi":"2025-07-31","saturated":true},{"slug":"fictionlivebench-60k-tokens","family":"fictionlivebench-60k-tokens","name":"Fiction.LiveBench - 60K tokens","nStates":53,"nModels":47,"difficulty":136.7,"saturation":155.3,"saturationExtrapolated":false,"frontierPct":97.8,"extrapolated":false,"extrapolation":null,"maxTested":155.7,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-02-11","saturated":true},{"slug":"concept-synth-induction-v1-ci","family":"concept-synth-induction-v1","name":"Concept Synth INDUCTION v1","nStates":12,"nModels":12,"difficulty":136.8,"saturation":193.2,"saturationExtrapolated":true,"frontierPct":83.6,"extrapolated":true,"extrapolation":"below","maxTested":156.3,"saturationDate":"2027-10-15","saturationDateLo":"2026-09-02","saturationDateHi":"2030-06-20","saturated":false},{"slug":"swe-bench-verified","family":"swe-bench-verified","name":"SWE-bench Verified","nStates":233,"nModels":169,"difficulty":136.8,"saturation":165.0,"saturationExtrapolated":false,"frontierPct":91.4,"extrapolated":false,"extrapolation":null,"maxTested":167.8,"saturationDate":"2026-07-24","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-01","saturated":true},{"slug":"ifbench-artificial-analysis","family":"ifbench-artificial-analysis","name":"IFBench (Artificial Analysis avg@5)","nStates":443,"nModels":324,"difficulty":137.2,"saturation":194.8,"saturationExtrapolated":true,"frontierPct":77.9,"extrapolated":false,"extrapolation":null,"maxTested":167.3,"saturationDate":"2028-03-30","saturationDateLo":"2028-01-02","saturationDateHi":"2028-07-03","saturated":false},{"slug":"longform-creative-writing","family":"longform-creative-writing","name":"Longform Creative Writing","nStates":122,"nModels":121,"difficulty":137.2,"saturation":178.8,"saturationExtrapolated":true,"frontierPct":83.1,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-06-10","saturationDateLo":"2027-03-19","saturationDateHi":"2027-09-02","saturated":false},{"slug":"compilebench","family":"compilebench","name":"CompileBench","nStates":25,"nModels":19,"difficulty":137.3,"saturation":157.5,"saturationExtrapolated":false,"frontierPct":97.6,"extrapolated":false,"extrapolation":null,"maxTested":150.4,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-04-08","saturated":true},{"slug":"spatialbench-v1-verified","family":"spatialbench-v1-verified","name":"SpatialBench V0 Verified","nStates":16,"nModels":16,"difficulty":137.3,"saturation":207.3,"saturationExtrapolated":true,"frontierPct":76.5,"extrapolated":true,"extrapolation":"below","maxTested":165.5,"saturationDate":"2028-09-06","saturationDateLo":"2027-10-03","saturationDateHi":"2031-01-21","saturated":false},{"slug":"edit-bench","family":"edit-bench","name":"EDIT-Bench Complete","nStates":41,"nModels":38,"difficulty":137.4,"saturation":237.9,"saturationExtrapolated":true,"frontierPct":66.4,"extrapolated":false,"extrapolation":null,"maxTested":149.8,"saturationDate":"2030-12-25","saturationDateLo":"2029-11-16","saturationDateHi":"2033-03-21","saturated":false},{"slug":"graphwalks-parents-up-to-128k","family":"graphwalks-parents-up-to-128k","name":"GraphWalks Parents: Up to 128K","nStates":18,"nModels":18,"difficulty":137.5,"saturation":165.5,"saturationExtrapolated":false,"frontierPct":92.1,"extrapolated":false,"extrapolation":null,"maxTested":158.8,"saturationDate":"2026-06-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-11-13","saturated":true},{"slug":"roboflow-vision-evals-2026-10","family":"roboflow-vision-evals-2026-10","name":"Roboflow Vision Evals (October 2026, five tasks)","nStates":41,"nModels":41,"difficulty":137.5,"saturation":204.6,"saturationExtrapolated":true,"frontierPct":75.0,"extrapolated":true,"extrapolation":"below","maxTested":167.2,"saturationDate":"2028-11-23","saturationDateLo":"2027-12-30","saturationDateHi":"2030-05-27","saturated":false},{"slug":"tau2-bench-airline-50","family":"tau2-bench-airline-50","name":"\u03c4\u00b2-bench Airline - 50 tasks","nStates":73,"nModels":58,"difficulty":137.5,"saturation":169.4,"saturationExtrapolated":true,"frontierPct":89.4,"extrapolated":false,"extrapolation":null,"maxTested":160.6,"saturationDate":"2026-09-02","saturationDateLo":"2026-07-20","saturationDateHi":"2026-12-21","saturated":true},{"slug":"gpqa-main","family":"gpqa-main","name":"GPQA Main","nStates":81,"nModels":79,"difficulty":137.7,"saturation":166.6,"saturationExtrapolated":false,"frontierPct":93.8,"extrapolated":false,"extrapolation":null,"maxTested":149.3,"saturationDate":"2026-06-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-02","saturated":true},{"slug":"step-game-trueskill-mu","family":"step-game-trueskill-mu","name":"Step Game TrueSkill Mu","nStates":64,"nModels":57,"difficulty":137.7,"saturation":161.7,"saturationExtrapolated":false,"frontierPct":95.3,"extrapolated":false,"extrapolation":null,"maxTested":155.6},{"slug":"elimination-game-trueskill-mu","family":"elimination-game-trueskill-mu","name":"Elimination Game TrueSkill Mu","nStates":58,"nModels":55,"difficulty":137.9,"saturation":171.7,"saturationExtrapolated":true,"frontierPct":88.4,"extrapolated":false,"extrapolation":null,"maxTested":153.8},{"slug":"aider-polyglot","family":"aider-polyglot","name":"Aider Polyglot","nStates":94,"nModels":71,"difficulty":138.1,"saturation":154.7,"saturationExtrapolated":false,"frontierPct":98.0,"extrapolated":false,"extrapolation":null,"maxTested":149.8,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2025-12-11","saturated":true},{"slug":"cyberbench-v1-1-overall","family":"cyberbench-v1-1-overall","name":"CyberBench v1.1 Overall","nStates":37,"nModels":36,"difficulty":138.1,"saturation":209.2,"saturationExtrapolated":true,"frontierPct":74.0,"extrapolated":false,"extrapolation":null,"maxTested":167.8,"saturationDate":"2029-02-03","saturationDateLo":"2028-01-01","saturationDateHi":"2031-02-04","saturated":false},{"slug":"eval-connections","family":"eval-connections","name":"Eval Connections","nStates":80,"nModels":80,"difficulty":138.1,"saturation":156.2,"saturationExtrapolated":false,"frontierPct":97.3,"extrapolated":false,"extrapolation":null,"maxTested":161.7,"saturationDate":"2025-12-11","saturationDateLo":"2025-07-31","saturationDateHi":"2026-03-05","saturated":true},{"slug":"wordlebench","family":"wordlebench","name":"WordleBench","nStates":92,"nModels":92,"difficulty":138.1,"saturation":168.3,"saturationExtrapolated":false,"frontierPct":92.5,"extrapolated":false,"extrapolation":null,"maxTested":164.2,"saturationDate":"2026-04-22","saturationDateLo":"2025-12-11","saturationDateHi":"2026-12-09","saturated":true},{"slug":"supergpqa","family":"supergpqa","name":"SuperGPQA","nStates":90,"nModels":80,"difficulty":138.4,"saturation":178.1,"saturationExtrapolated":true,"frontierPct":85.8,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2027-04-21","saturationDateLo":"2027-02-25","saturationDateHi":"2027-06-09","saturated":false},{"slug":"healthbench-card","family":"healthbench-card","name":"HealthBench","nStates":45,"nModels":39,"difficulty":138.6,"saturation":273.6,"saturationExtrapolated":true,"frontierPct":62.2,"extrapolated":false,"extrapolation":null,"maxTested":168.7,"saturationDate":"2033-03-19","saturationDateLo":"2031-09-19","saturationDateHi":"2035-05-07","saturated":false},{"slug":"omni-math-helm-capabilities","family":"omni-math-helm-capabilities","name":"Omni-MATH (HELM Capabilities)","nStates":58,"nModels":56,"difficulty":138.7,"saturation":182.7,"saturationExtrapolated":true,"frontierPct":81.4,"extrapolated":false,"extrapolation":null,"maxTested":149.3,"saturationDate":"2027-08-19","saturationDateLo":"2027-03-23","saturationDateHi":"2028-02-16","saturated":false},{"slug":"openai-mrcr-2-needle-128k","family":"openai-mrcr-2-needle-128k","name":"OpenAI-MRCR 2-needle - 128K","nStates":13,"nModels":13,"difficulty":138.7,"saturation":187.3,"saturationExtrapolated":true,"frontierPct":83.9,"extrapolated":false,"extrapolation":null,"maxTested":150.2,"saturationDate":"2027-06-08","saturationDateLo":"2026-04-07","saturationDateHi":"2029-11-30","saturated":false},{"slug":"eq-bench-3","family":"eq-bench-3","name":"EQ-Bench 3","nStates":76,"nModels":76,"difficulty":138.8,"saturation":172.6,"saturationExtrapolated":true,"frontierPct":86.5,"extrapolated":false,"extrapolation":null,"maxTested":164.6},{"slug":"cad-eval","family":"cad-eval","name":"CadEval","nStates":17,"nModels":17,"difficulty":139.4,"saturation":166.9,"saturationExtrapolated":false,"frontierPct":91.7,"extrapolated":false,"extrapolation":null,"maxTested":147.5,"saturationDate":"2026-07-09","saturationDateLo":"2026-02-11","saturationDateHi":"2027-02-24","saturated":true},{"slug":"hermes-bench","family":"hermes-bench","name":"Hermes Bench","nStates":15,"nModels":15,"difficulty":139.6,"saturation":196.1,"saturationExtrapolated":true,"frontierPct":76.2,"extrapolated":true,"extrapolation":"below","maxTested":167.3,"saturationDate":"2028-05-28","saturationDateLo":"2027-11-01","saturationDateHi":"2029-08-21","saturated":false},{"slug":"macroscopebench","family":"macroscopebench","name":"MacroscopeBench","nStates":45,"nModels":14,"difficulty":139.6,"saturation":191.6,"saturationExtrapolated":true,"frontierPct":81.2,"extrapolated":false,"extrapolation":null,"maxTested":174.4,"saturationDate":"2027-11-07","saturationDateLo":"2027-06-11","saturationDateHi":"2028-06-08","saturated":false},{"slug":"mmlongbench-doc","family":"mmlongbench-doc","name":"MMLongBench-Doc","nStates":62,"nModels":50,"difficulty":139.8,"saturation":201.8,"saturationExtrapolated":true,"frontierPct":74.6,"extrapolated":false,"extrapolation":null,"maxTested":159.5,"saturationDate":"2028-09-16","saturationDateLo":"2028-04-24","saturationDateHi":"2029-03-05","saturated":false},{"slug":"thematic-generalization-v2","family":"thematic-generalization-v2","name":"Thematic Generalization V2","nStates":30,"nModels":23,"difficulty":139.9,"saturation":160.8,"saturationExtrapolated":false,"frontierPct":96.5,"extrapolated":false,"extrapolation":null,"maxTested":161.5,"saturationDate":"2026-02-04","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"aa-lcr-1-1-artificial-analysis","family":"aa-lcr-1-1-artificial-analysis","name":"AA-LCR v1.1","nStates":592,"nModels":390,"difficulty":140.4,"saturation":168.5,"saturationExtrapolated":false,"frontierPct":90.2,"extrapolated":false,"extrapolation":null,"maxTested":169.8,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2026-09-02","saturated":true},{"slug":"big-bench-extra-hard","family":"big-bench-extra-hard","name":"BIG-Bench Extra Hard","nStates":17,"nModels":15,"difficulty":140.5,"saturation":159.6,"saturationExtrapolated":false,"frontierPct":96.3,"extrapolated":false,"extrapolation":null,"maxTested":147.6,"saturationDate":"2026-02-11","saturationDateLo":"2026-01-30","saturationDateHi":"2026-03-05","saturated":true},{"slug":"clinicare-bench","family":"clinicare-bench","name":"CliniCARE-Bench","nStates":12,"nModels":12,"difficulty":140.9,"saturation":214.8,"saturationExtrapolated":true,"frontierPct":78.7,"extrapolated":true,"extrapolation":"below","maxTested":164.6,"saturationDate":"2029-05-22","saturationDateLo":"2028-04-11","saturationDateHi":"2031-11-14","saturated":false},{"slug":"polymath","family":"polymath","name":"PolyMath","nStates":95,"nModels":85,"difficulty":141.0,"saturation":172.8,"saturationExtrapolated":true,"frontierPct":88.9,"extrapolated":false,"extrapolation":null,"maxTested":156.8,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2026-11-23","saturated":true},{"slug":"stembench120-1-0","family":"stembench120-1-0","name":"STEMBench120 v1.0.0 (closed-book)","nStates":8,"nModels":8,"difficulty":141.0,"saturation":233.0,"saturationExtrapolated":true,"frontierPct":69.5,"extrapolated":true,"extrapolation":"below","maxTested":167.2,"saturationDate":"2030-08-01","saturationDateLo":"2028-05-14","saturationDateHi":"2034-06-09","saturated":false},{"slug":"multichallenge-scale-ai-2","family":"multichallenge-scale-ai-2","name":"MultiChallenge","nStates":70,"nModels":66,"difficulty":141.2,"saturation":188.9,"saturationExtrapolated":true,"frontierPct":77.5,"extrapolated":false,"extrapolation":null,"maxTested":165.4,"saturationDate":"2028-01-16","saturationDateLo":"2027-10-03","saturationDateHi":"2028-05-23","saturated":false},{"slug":"veracode-genai-code-security-fall-2026","family":"veracode-genai-code-security-fall-2026","name":"Veracode GenAI Code Security (Fall 2026)","nStates":16,"nModels":16,"difficulty":141.2,"saturation":209.9,"saturationExtrapolated":true,"frontierPct":71.6,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2029-04-16","saturationDateLo":"2028-05-29","saturationDateHi":"2030-05-19","saturated":false},{"slug":"livecodebench-pro","family":"livecodebench-pro","name":"LiveCodeBench Pro","nStates":42,"nModels":34,"difficulty":141.3,"saturation":164.5,"saturationExtrapolated":false,"frontierPct":93.1,"extrapolated":false,"extrapolation":null,"maxTested":157.3},{"slug":"lvbench","family":"lvbench","name":"LVBench","nStates":73,"nModels":66,"difficulty":141.4,"saturation":188.9,"saturationExtrapolated":true,"frontierPct":83.4,"extrapolated":false,"extrapolation":null,"maxTested":167.5,"saturationDate":"2027-12-28","saturationDateLo":"2027-08-17","saturationDateHi":"2028-05-27","saturated":false},{"slug":"apex-v1-extended","family":"apex-v1-extended","name":"APEX-1 (Mercor, mean score)","nStates":68,"nModels":65,"difficulty":141.7,"saturation":205.1,"saturationExtrapolated":true,"frontierPct":71.8,"extrapolated":false,"extrapolation":null,"maxTested":169.0,"saturationDate":"2029-01-07","saturationDateLo":"2028-08-03","saturationDateHi":"2029-09-26","saturated":false},{"slug":"tau-bench-airline","family":"tau-bench-airline","name":"\u03c4-bench (Airline)","nStates":55,"nModels":40,"difficulty":141.7,"saturation":179.7,"saturationExtrapolated":true,"frontierPct":82.4,"extrapolated":false,"extrapolation":null,"maxTested":148.4,"saturationDate":"2027-05-28","saturationDateLo":"2027-01-22","saturationDateHi":"2027-11-19","saturated":false},{"slug":"swe-bench-multilingual","family":"swe-bench-multilingual","name":"SWE-bench Multilingual","nStates":108,"nModels":92,"difficulty":141.8,"saturation":165.1,"saturationExtrapolated":false,"frontierPct":91.7,"extrapolated":false,"extrapolation":null,"maxTested":167.8,"saturationDate":"2026-07-24","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-01","saturated":true},{"slug":"chessbench-ai-coherence","family":"chessbench-ai-coherence","name":"ChessBench Coherence","nStates":200,"nModels":65,"difficulty":142.3,"saturation":188.5,"saturationExtrapolated":true,"frontierPct":82.3,"extrapolated":false,"extrapolation":null,"maxTested":175.2,"saturationDate":"2027-06-03","saturationDateLo":"2027-02-22","saturationDateHi":"2027-10-13","saturated":false},{"slug":"immigration-bench","family":"immigration-bench","name":"Immigration Bench","nStates":24,"nModels":23,"difficulty":142.5,"saturation":205.3,"saturationExtrapolated":true,"frontierPct":71.0,"extrapolated":true,"extrapolation":"below","maxTested":168.0,"saturationDate":"2029-01-20","saturationDateLo":"2028-07-24","saturationDateHi":"2029-12-04","saturated":false},{"slug":"cnmo-2024","family":"cnmo-2024","name":"CNMO 2024","nStates":11,"nModels":11,"difficulty":142.6,"saturation":164.4,"saturationExtrapolated":false,"frontierPct":98.9,"extrapolated":false,"extrapolation":null,"maxTested":159.5,"saturationDate":"2025-03-24","saturationDateLo":"2025-01-30","saturationDateHi":"2025-07-31","saturated":true},{"slug":"screenspot-pro","family":"screenspot-pro","name":"ScreenSpot-Pro","nStates":65,"nModels":47,"difficulty":142.6,"saturation":196.0,"saturationExtrapolated":true,"frontierPct":75.2,"extrapolated":false,"extrapolation":null,"maxTested":170.0,"saturationDate":"2028-04-18","saturationDateLo":"2027-08-31","saturationDateHi":"2029-03-07","saturated":false},{"slug":"creative-writing-v3-elo","family":"creative-writing-v3-elo","name":"Creative Writing v3 Elo","nStates":121,"nModels":121,"difficulty":142.7,"saturation":182.0,"saturationExtrapolated":true,"frontierPct":79.7,"extrapolated":false,"extrapolation":null,"maxTested":167.2},{"slug":"mcp-atlas-april-2026-all-1000","family":"mcp-atlas-april-2026-all-1000","name":"MCP Atlas April 2026 (all 1,000 tasks)","nStates":42,"nModels":36,"difficulty":143.2,"saturation":167.1,"saturationExtrapolated":false,"frontierPct":90.5,"extrapolated":false,"extrapolation":null,"maxTested":166.1,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2026-11-09","saturated":true},{"slug":"warc-bench-test-192","family":"warc-bench-test-192","name":"WARC-Bench (192-task test split)","nStates":8,"nModels":8,"difficulty":143.2,"saturation":186.9,"saturationExtrapolated":true,"frontierPct":85.4,"extrapolated":true,"extrapolation":"below","maxTested":170.5,"saturationDate":"2027-07-03","saturationDateLo":"2026-11-06","saturationDateHi":"2029-02-18","saturated":false},{"slug":"ramp-swe-bench-observed-78","family":"ramp-swe-bench-observed-78","name":"Ramp SWE-Bench (78 published tasks)","nStates":25,"nModels":25,"difficulty":143.6,"saturation":167.2,"saturationExtrapolated":false,"frontierPct":90.8,"extrapolated":true,"extrapolation":"below","maxTested":164.5,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2026-09-02","saturated":true},{"slug":"public-benefits-bench","family":"public-benefits-bench","name":"Public Benefits Bench v1.1","nStates":47,"nModels":47,"difficulty":143.8,"saturation":204.0,"saturationExtrapolated":true,"frontierPct":70.3,"extrapolated":false,"extrapolation":null,"maxTested":167.1,"saturationDate":"2029-02-06","saturationDateLo":"2028-09-08","saturationDateHi":"2029-08-22","saturated":false},{"slug":"ahk-eval","family":"ahk-eval","name":"AHK-Eval","nStates":35,"nModels":34,"difficulty":143.9,"saturation":162.0,"saturationExtrapolated":false,"frontierPct":95.2,"extrapolated":false,"extrapolation":null,"maxTested":168.1,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-11","saturationDateHi":"2026-04-07","saturated":true},{"slug":"cred-v0-92","family":"cred-v0-92","name":"CRED v0.92","nStates":41,"nModels":23,"difficulty":144.0,"saturation":161.7,"saturationExtrapolated":false,"frontierPct":95.1,"extrapolated":false,"extrapolation":null,"maxTested":166.1,"saturationDate":"2026-03-05","saturationDateLo":"2026-02-11","saturationDateHi":"2026-04-07","saturated":true},{"slug":"cyberarena","family":"cyberarena","name":"CyberArena","nStates":11,"nModels":11,"difficulty":144.0,"saturation":202.0,"saturationExtrapolated":true,"frontierPct":76.1,"extrapolated":true,"extrapolation":"below","maxTested":164.6,"saturationDate":"2028-07-31","saturationDateLo":"2027-06-21","saturationDateHi":"2030-11-02","saturated":false},{"slug":"sage-v1-1","family":"sage-v1-1","name":"SAGE v1.1","nStates":25,"nModels":25,"difficulty":144.0,"saturation":180.4,"saturationExtrapolated":true,"frontierPct":90.4,"extrapolated":false,"extrapolation":null,"maxTested":168.3,"saturationDate":"2027-07-11","saturationDateLo":"2027-03-31","saturationDateHi":"2027-11-02","saturated":false},{"slug":"eval-connections-one-shot-traps","family":"eval-connections-one-shot-traps","name":"Eval Connections One-Shot with Traps","nStates":58,"nModels":58,"difficulty":144.1,"saturation":174.0,"saturationExtrapolated":true,"frontierPct":85.9,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-01-11","saturationDateLo":"2026-09-02","saturationDateHi":"2027-04-26","saturated":false},{"slug":"deepsearchqa-f1-score","family":"deepsearchqa-f1-score","name":"DeepSearchQA (F1 score)","nStates":42,"nModels":29,"difficulty":144.2,"saturation":161.5,"saturationExtrapolated":false,"frontierPct":95.2,"extrapolated":false,"extrapolation":null,"maxTested":164.6,"saturationDate":"2026-03-05","saturationDateLo":"2026-02-11","saturationDateHi":"2026-04-07","saturated":true},{"slug":"browsecomp-zh","family":"browsecomp-zh","name":"BrowseComp-ZH","nStates":20,"nModels":20,"difficulty":144.3,"saturation":166.1,"saturationExtrapolated":false,"frontierPct":92.5,"extrapolated":false,"extrapolation":null,"maxTested":152.7,"saturationDate":"2026-06-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-11-29","saturated":true},{"slug":"kimi-code-bench-v2","family":"kimi-code-bench-v2","name":"Kimi Code Bench v2","nStates":9,"nModels":8,"difficulty":144.3,"saturation":192.3,"saturationExtrapolated":true,"frontierPct":78.3,"extrapolated":true,"extrapolation":"below","maxTested":165.6,"saturationDate":"2027-12-11","saturationDateLo":"2027-04-21","saturationDateHi":"2029-05-17","saturated":false},{"slug":"mrcr-v2-8-needle-128k-average","family":"mrcr-v2-8-needle-128k-average","name":"MRCR v2 (8-needle) - 128K average","nStates":54,"nModels":46,"difficulty":144.3,"saturation":166.3,"saturationExtrapolated":false,"frontierPct":92.1,"extrapolated":false,"extrapolation":null,"maxTested":163.1,"saturationDate":"2026-06-09","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-02","saturated":true},{"slug":"tutorbench-scale-ai-2","family":"tutorbench-scale-ai-2","name":"TutorBench","nStates":26,"nModels":20,"difficulty":144.3,"saturation":254.2,"saturationExtrapolated":true,"frontierPct":62.4,"extrapolated":false,"extrapolation":null,"maxTested":160.1,"saturationDate":"2031-12-03","saturationDateLo":"2030-03-04","saturationDateHi":"2034-08-22","saturated":false},{"slug":"ide-bench","family":"ide-bench","name":"IDE-Bench","nStates":14,"nModels":14,"difficulty":144.4,"saturation":160.0,"saturationExtrapolated":false,"frontierPct":97.0,"extrapolated":false,"extrapolation":null,"maxTested":152.7,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-07-24","saturated":true},{"slug":"graphwalks-bfs-256k","family":"graphwalks-bfs-256k","name":"GraphWalks BFS - 256K","nStates":14,"nModels":11,"difficulty":144.9,"saturation":169.3,"saturationExtrapolated":true,"frontierPct":92.8,"extrapolated":true,"extrapolation":"below","maxTested":164.0,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2027-04-28","saturated":true},{"slug":"lmarena-vision-arena","family":"lmarena-vision-arena","name":"Vision Arena Overall","nStates":113,"nModels":103,"difficulty":145.1,"saturation":181.2,"saturationExtrapolated":true,"frontierPct":79.4,"extrapolated":false,"extrapolation":null,"maxTested":167.1},{"slug":"the-agent-company","family":"the-agent-company","name":"The Agent Company","nStates":13,"nModels":13,"difficulty":145.2,"saturation":167.2,"saturationExtrapolated":true,"frontierPct":90.4,"extrapolated":true,"extrapolation":"above","maxTested":143.9,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2027-01-09","saturated":true},{"slug":"cybench","family":"cybench","name":"Cybench","nStates":13,"nModels":13,"difficulty":145.4,"saturation":165.2,"saturationExtrapolated":false,"frontierPct":92.6,"extrapolated":false,"extrapolation":null,"maxTested":159.0,"saturationDate":"2026-06-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-11-13","saturated":true},{"slug":"vibevideobench","family":"vibevideobench","name":"VibeVideoBench (overall quality, no video = 0)","nStates":8,"nModels":8,"difficulty":145.4,"saturation":193.8,"saturationExtrapolated":true,"frontierPct":79.5,"extrapolated":true,"extrapolation":"below","maxTested":168.9,"saturationDate":"2028-02-03","saturationDateLo":"2027-04-19","saturationDateHi":"2030-01-05","saturated":false},{"slug":"public-benefits-bench-both","family":"public-benefits-bench-both","name":"Public Benefits Bench v1 - Both","nStates":13,"nModels":13,"difficulty":145.6,"saturation":214.6,"saturationExtrapolated":true,"frontierPct":67.8,"extrapolated":false,"extrapolation":null,"maxTested":164.0,"saturationDate":"2029-06-04","saturationDateLo":"2028-06-29","saturationDateHi":"2031-06-24","saturated":false},{"slug":"poker_agent","family":"poker_agent","name":"Poker Agent","nStates":17,"nModels":17,"difficulty":145.7,"saturation":165.5,"saturationExtrapolated":false,"frontierPct":93.6,"extrapolated":false,"extrapolation":null,"maxTested":154.3},{"slug":"llm-chess-elo","family":"llm-chess-elo","name":"LLM Chess Elo","nStates":135,"nModels":94,"difficulty":145.8,"saturation":179.1,"saturationExtrapolated":true,"frontierPct":82.6,"extrapolated":false,"extrapolation":null,"maxTested":169.6},{"slug":"solo-bench-easy","family":"solo-bench-easy","name":"SOLO Bench Easy","nStates":21,"nModels":21,"difficulty":145.8,"saturation":159.9,"saturationExtrapolated":false,"frontierPct":97.6,"extrapolated":false,"extrapolation":null,"maxTested":147.1,"saturationDate":"2026-02-04","saturationDateLo":"2025-07-31","saturationDateHi":"2026-06-09","saturated":true},{"slug":"culturalriddles","family":"culturalriddles","name":"CulturalRiddles","nStates":15,"nModels":14,"difficulty":146.1,"saturation":181.6,"saturationExtrapolated":true,"frontierPct":79.2,"extrapolated":false,"extrapolation":null,"maxTested":158.0,"saturationDate":"2027-07-25","saturationDateLo":"2027-01-29","saturationDateHi":"2028-04-13","saturated":false},{"slug":"motionbench-dev","family":"motionbench-dev","name":"MotionBench (dev)","nStates":9,"nModels":8,"difficulty":146.1,"saturation":247.6,"saturationExtrapolated":true,"frontierPct":72.4,"extrapolated":false,"extrapolation":null,"maxTested":152.1,"saturationDate":"2031-03-23","saturationDateLo":"2029-03-02","saturationDateHi":"2035-09-14","saturated":false},{"slug":"astabench-2026-core-hard","family":"astabench-2026-core-hard","name":"CORE-Bench-Hard (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":146.6,"saturation":160.9,"saturationExtrapolated":false,"frontierPct":97.1,"extrapolated":false,"extrapolation":null,"maxTested":163.2,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"erdosbench","family":"erdosbench","name":"ErdosBench","nStates":22,"nModels":22,"difficulty":146.6,"saturation":181.2,"saturationExtrapolated":true,"frontierPct":80.6,"extrapolated":true,"extrapolation":"below","maxTested":168.1,"saturationDate":"2027-06-28","saturationDateLo":"2027-03-04","saturationDateHi":"2028-01-08","saturated":false},{"slug":"apex-shortlist-matharena","family":"apex-shortlist-matharena","name":"Apex Shortlist","nStates":52,"nModels":45,"difficulty":146.7,"saturation":162.0,"saturationExtrapolated":false,"frontierPct":96.5,"extrapolated":false,"extrapolation":null,"maxTested":163.9,"saturationDate":"2026-02-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"browsecomp","family":"browsecomp","name":"BrowseComp","nStates":124,"nModels":70,"difficulty":146.8,"saturation":164.5,"saturationExtrapolated":false,"frontierPct":93.8,"extrapolated":false,"extrapolation":null,"maxTested":169.4,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-06-09","saturated":true},{"slug":"cresta-bench","family":"cresta-bench","name":"Cresta-Bench","nStates":14,"nModels":14,"difficulty":147.1,"saturation":200.6,"saturationExtrapolated":true,"frontierPct":76.0,"extrapolated":false,"extrapolation":null,"maxTested":160.3,"saturationDate":"2028-02-27","saturationDateLo":"2027-01-05","saturationDateHi":"2030-10-23","saturated":false},{"slug":"mrcr-v2-8-needle-256k-512k","family":"mrcr-v2-8-needle-256k-512k","name":"OpenAI MRCR v2 8-needle - 256K-512K","nStates":10,"nModels":8,"difficulty":147.1,"saturation":169.6,"saturationExtrapolated":true,"frontierPct":98.6,"extrapolated":true,"extrapolation":"below","maxTested":168.1,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2027-07-24","saturated":true},{"slug":"osworld-verified","family":"os-world","name":"OSWorld","nStates":80,"nModels":53,"difficulty":147.1,"saturation":168.3,"saturationExtrapolated":false,"frontierPct":90.1,"extrapolated":false,"extrapolation":null,"maxTested":164.6,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-01","saturationDateHi":"2026-10-28","saturated":true},{"slug":"snorkelspatial","family":"snorkelspatial","name":"SnorkelSpatial","nStates":32,"nModels":32,"difficulty":147.3,"saturation":184.3,"saturationExtrapolated":true,"frontierPct":79.5,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2027-07-08","saturationDateLo":"2026-09-02","saturationDateHi":"2028-10-28","saturated":false},{"slug":"arc-agi-1-semi-private","family":"arc-agi-1-semi-private","name":"ARC-AGI-1 Semi-Private","nStates":274,"nModels":89,"difficulty":147.5,"saturation":160.0,"saturationExtrapolated":false,"frontierPct":98.1,"extrapolated":false,"extrapolation":null,"maxTested":171.7,"saturationDate":"2025-12-11","saturationDateLo":"2025-12-11","saturationDateHi":"2026-02-11","saturated":true},{"slug":"medxpertqa-mm-200","family":"medxpertqa-mm-200","name":"MedXpertQA MM (200-question set)","nStates":41,"nModels":39,"difficulty":147.5,"saturation":189.8,"saturationExtrapolated":true,"frontierPct":82.6,"extrapolated":true,"extrapolation":"below","maxTested":173.0,"saturationDate":"2027-10-02","saturationDateLo":"2027-05-24","saturationDateHi":"2028-06-10","saturated":false},{"slug":"gdpval","family":"gdpval","name":"GDPval","nStates":11,"nModels":10,"difficulty":147.8,"saturation":168.4,"saturationExtrapolated":true,"frontierPct":92.0,"extrapolated":true,"extrapolation":"below","maxTested":163.3,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2027-01-17","saturated":true},{"slug":"mcp-atlas","family":"mcp-atlas","name":"MCP Atlas (original protocol)","nStates":30,"nModels":27,"difficulty":147.8,"saturation":172.8,"saturationExtrapolated":true,"frontierPct":87.8,"extrapolated":false,"extrapolation":null,"maxTested":158.1,"saturationDate":"2026-09-12","saturationDateLo":"2026-04-22","saturationDateHi":"2027-08-29","saturated":true},{"slug":"medxpertqa-mm","family":"medxpertqa-mm","name":"MedXpertQA MM","nStates":43,"nModels":40,"difficulty":147.8,"saturation":170.8,"saturationExtrapolated":true,"frontierPct":90.2,"extrapolated":false,"extrapolation":null,"maxTested":161.6,"saturationDate":"2026-11-09","saturationDateLo":"2026-09-02","saturationDateHi":"2027-01-14","saturated":false},{"slug":"vsi-bench","family":"vsi-bench","name":"VSI-Bench","nStates":9,"nModels":8,"difficulty":147.8,"saturation":267.6,"saturationExtrapolated":true,"frontierPct":61.1,"extrapolated":false,"extrapolation":null,"maxTested":151.6,"saturationDate":"2031-12-08","saturationDateLo":"2028-11-02","saturationDateHi":"2038-06-06","saturated":false},{"slug":"design-arena-overall-code-categories","family":"design-arena-overall-code-categories","name":"Design Arena Overall Code Categories","nStates":170,"nModels":150,"difficulty":147.9,"saturation":175.2,"saturationExtrapolated":true,"frontierPct":82.6,"extrapolated":false,"extrapolation":null,"maxTested":167.2},{"slug":"drone-bench","family":"drone-bench","name":"Drone-Bench","nStates":18,"nModels":18,"difficulty":148.1,"saturation":168.7,"saturationExtrapolated":true,"frontierPct":89.9,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2026-12-09","saturated":true},{"slug":"judgemark-v4","family":"judgemark-v4","name":"Judgemark v4","nStates":41,"nModels":41,"difficulty":148.2,"saturation":173.9,"saturationExtrapolated":true,"frontierPct":84.5,"extrapolated":false,"extrapolation":null,"maxTested":164.6,"saturationDate":"2027-01-27","saturationDateLo":"2026-09-02","saturationDateHi":"2027-07-18","saturated":false},{"slug":"pact-negotiation-bilateral-rating","family":"pact-negotiation-bilateral-rating","name":"PACT Negotiation Bilateral Rating","nStates":25,"nModels":25,"difficulty":148.4,"saturation":174.9,"saturationExtrapolated":true,"frontierPct":85.7,"extrapolated":false,"extrapolation":null,"maxTested":164.0},{"slug":"terminal-bench-2-1","family":"terminal-bench-2-1","name":"Terminal-Bench 2.1","nStates":330,"nModels":213,"difficulty":148.7,"saturation":166.9,"saturationExtrapolated":false,"frontierPct":90.5,"extrapolated":false,"extrapolation":null,"maxTested":168.2,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-01","saturationDateHi":"2026-09-02","saturated":true},{"slug":"facts-benchmark-suite","family":"facts-benchmark-suite","name":"FACTS Benchmark Suite","nStates":29,"nModels":28,"difficulty":148.9,"saturation":192.9,"saturationExtrapolated":true,"frontierPct":72.9,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2028-03-30","saturationDateLo":"2027-09-11","saturationDateHi":"2029-03-29","saturated":false},{"slug":"buyout-game-bradley-terry","family":"buyout-game-bradley-terry","name":"Buyout Game Bradley-Terry Rating","nStates":36,"nModels":32,"difficulty":149.1,"saturation":168.7,"saturationExtrapolated":true,"frontierPct":89.2,"extrapolated":false,"extrapolation":null,"maxTested":159.8},{"slug":"finance-agent-v2-1","family":"finance-agent-v2-1","name":"Finance Agent v2.1","nStates":32,"nModels":32,"difficulty":149.1,"saturation":208.2,"saturationExtrapolated":true,"frontierPct":66.8,"extrapolated":true,"extrapolation":"below","maxTested":167.7,"saturationDate":"2029-04-21","saturationDateLo":"2028-09-09","saturationDateHi":"2030-03-19","saturated":false},{"slug":"diligence-stack-agent","family":"diligence-stack-agent","name":"Diligence Stack Agent Bench","nStates":16,"nModels":14,"difficulty":149.2,"saturation":167.4,"saturationExtrapolated":false,"frontierPct":91.6,"extrapolated":false,"extrapolation":null,"maxTested":163.9,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-02","saturated":true},{"slug":"financeqa","family":"financeqa","name":"FinanceQA","nStates":19,"nModels":19,"difficulty":149.7,"saturation":222.5,"saturationExtrapolated":true,"frontierPct":64.6,"extrapolated":true,"extrapolation":"above","maxTested":147.1,"saturationDate":"2029-12-31","saturationDateLo":"2028-08-10","saturationDateHi":"2032-12-26","saturated":false},{"slug":"emb-v1-1","family":"emb-v1-1","name":"Excel Modeling Benchmark v1.1","nStates":36,"nModels":36,"difficulty":149.8,"saturation":181.9,"saturationExtrapolated":true,"frontierPct":76.8,"extrapolated":true,"extrapolation":"below","maxTested":167.7,"saturationDate":"2027-08-24","saturationDateLo":"2027-06-11","saturationDateHi":"2027-11-27","saturated":false},{"slug":"ojbench-cpp","family":"ojbench-cpp","name":"OJBench (C++)","nStates":9,"nModels":9,"difficulty":149.8,"saturation":173.4,"saturationExtrapolated":true,"frontierPct":88.5,"extrapolated":false,"extrapolation":null,"maxTested":156.1,"saturationDate":"2026-09-02","saturationDateLo":"2026-04-07","saturationDateHi":"2027-05-30","saturated":true},{"slug":"chessbench-ai-elo","family":"chessbench-ai-elo","name":"ChessBench Elo","nStates":200,"nModels":65,"difficulty":149.9,"saturation":186.0,"saturationExtrapolated":true,"frontierPct":78.8,"extrapolated":false,"extrapolation":null,"maxTested":171.8},{"slug":"extended-nyt-connections-940-quadratic-v1","family":"extended-nyt-connections-940-quadratic-v1","name":"Extended NYT Connections 940 (Quadratic v1)","nStates":116,"nModels":87,"difficulty":149.9,"saturation":164.8,"saturationExtrapolated":false,"frontierPct":94.8,"extrapolated":false,"extrapolation":null,"maxTested":169.8,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-06-09","saturated":true},{"slug":"cadqueryeval-corrected-geometry","family":"cadqueryeval-corrected-geometry","name":"CadQueryEval (Corrected Geometry)","nStates":86,"nModels":86,"difficulty":150.2,"saturation":172.6,"saturationExtrapolated":true,"frontierPct":85.2,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2026-12-30","saturationDateLo":"2026-09-02","saturationDateHi":"2027-03-27","saturated":false},{"slug":"learn2play-bench","family":"learn2play-bench","name":"Learn2Play Bench","nStates":9,"nModels":9,"difficulty":150.2,"saturation":185.8,"saturationExtrapolated":true,"frontierPct":77.4,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-08-14","saturationDateLo":"2027-02-28","saturationDateHi":"2028-04-30","saturated":false},{"slug":"where2place","family":"where2place","name":"Where2Place","nStates":28,"nModels":24,"difficulty":150.2,"saturation":234.6,"saturationExtrapolated":true,"frontierPct":62.8,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2030-07-21","saturationDateLo":"2029-01-29","saturationDateHi":"2032-12-13","saturated":false},{"slug":"scale-seal-visual-language-understanding","family":"scale-seal-visual-language-understanding","name":"VISTA","nStates":61,"nModels":52,"difficulty":150.3,"saturation":230.0,"saturationExtrapolated":true,"frontierPct":62.2,"extrapolated":false,"extrapolation":null,"maxTested":160.1,"saturationDate":"2030-09-02","saturationDateLo":"2030-02-16","saturationDateHi":"2031-06-30","saturated":false},{"slug":"babaisbench-intro","family":"babaisbench-intro","name":"BabaIsBench Stage 0: The Intro","nStates":30,"nModels":30,"difficulty":150.4,"saturation":166.2,"saturationExtrapolated":false,"frontierPct":95.1,"extrapolated":false,"extrapolation":null,"maxTested":166.1,"saturationDate":"2026-06-09","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-26","saturated":true},{"slug":"terminal-bench-2-0","family":"terminal-bench-2-0","name":"Terminal-Bench 2.0","nStates":148,"nModels":87,"difficulty":150.4,"saturation":168.8,"saturationExtrapolated":true,"frontierPct":88.2,"extrapolated":false,"extrapolation":null,"maxTested":163.3,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2026-11-29","saturated":true},{"slug":"live-bench-2","family":"live-bench-2","name":"LiveBench 2.0","nStates":53,"nModels":53,"difficulty":150.8,"saturation":175.1,"saturationExtrapolated":true,"frontierPct":82.5,"extrapolated":false,"extrapolation":null,"maxTested":168.5,"saturationDate":"2027-03-02","saturationDateLo":"2026-12-30","saturationDateHi":"2027-04-24","saturated":false},{"slug":"afantasia-valid-retry","family":"afantasia-valid-retry","name":"A-Fantasia (Valid Attempts with Retries)","nStates":56,"nModels":56,"difficulty":150.9,"saturation":232.4,"saturationExtrapolated":true,"frontierPct":63.3,"extrapolated":false,"extrapolation":null,"maxTested":161.6,"saturationDate":"2030-06-26","saturationDateLo":"2028-03-10","saturationDateHi":"2034-05-02","saturated":false},{"slug":"ale-bench","family":"ale-bench","name":"ALE-Bench","nStates":136,"nModels":122,"difficulty":150.9,"saturation":174.9,"saturationExtrapolated":true,"frontierPct":83.0,"extrapolated":false,"extrapolation":null,"maxTested":168.7},{"slug":"snorkel-finance-reasoning","family":"snorkel-finance-reasoning","name":"Snorkel Finance Reasoning","nStates":25,"nModels":25,"difficulty":150.9,"saturation":192.9,"saturationExtrapolated":true,"frontierPct":71.5,"extrapolated":false,"extrapolation":null,"maxTested":156.3,"saturationDate":"2028-03-09","saturationDateLo":"2027-08-12","saturationDateHi":"2029-02-12","saturated":false},{"slug":"spatialbench-v2","family":"spatialbench-v2","name":"SpatialBench V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":150.9,"saturation":189.4,"saturationExtrapolated":true,"frontierPct":73.8,"extrapolated":true,"extrapolation":"below","maxTested":167.2,"saturationDate":"2027-12-20","saturationDateLo":"2027-07-17","saturationDateHi":"2028-07-20","saturated":false},{"slug":"swe-bench-pro-public-dataset-scale-ai-2","family":"swe-bench-pro-public-dataset-scale-ai-2","name":"SWE-Bench Pro (Public Dataset)","nStates":119,"nModels":70,"difficulty":151.0,"saturation":180.3,"saturationExtrapolated":true,"frontierPct":77.2,"extrapolated":false,"extrapolation":null,"maxTested":164.9,"saturationDate":"2027-07-18","saturationDateLo":"2027-05-10","saturationDateHi":"2027-09-24","saturated":false},{"slug":"toolathlon-verified","family":"toolathlon-verified","name":"Toolathlon-Verified","nStates":47,"nModels":38,"difficulty":151.0,"saturation":170.2,"saturationExtrapolated":true,"frontierPct":86.5,"extrapolated":false,"extrapolation":null,"maxTested":164.9,"saturationDate":"2026-11-23","saturationDateLo":"2026-09-02","saturationDateHi":"2027-01-12","saturated":false},{"slug":"weirdml","family":"weirdml","name":"WeirdML (v2)","nStates":147,"nModels":119,"difficulty":151.0,"saturation":172.7,"saturationExtrapolated":true,"frontierPct":87.0,"extrapolated":false,"extrapolation":null,"maxTested":169.8,"saturationDate":"2027-01-12","saturationDateLo":"2026-11-14","saturationDateHi":"2027-03-03","saturated":false},{"slug":"musr-openllm-harness","family":"musr-openllm-harness","name":"MuSR (Open LLM Leaderboard v2)","nStates":121,"nModels":121,"difficulty":151.1,"saturation":450.1,"saturationExtrapolated":true,"frontierPct":53.2,"extrapolated":true,"extrapolation":"above","maxTested":126.1,"saturationDate":"2044-10-10","saturationDateLo":"2041-09-30","saturationDateHi":"2048-02-14","saturated":false},{"slug":"snorkel-agentic-coding","family":"snorkel-agentic-coding","name":"Snorkel Agentic Coding","nStates":11,"nModels":11,"difficulty":151.2,"saturation":176.8,"saturationExtrapolated":true,"frontierPct":82.0,"extrapolated":false,"extrapolation":null,"maxTested":155.7,"saturationDate":"2027-02-10","saturationDateLo":"2026-09-02","saturationDateHi":"2027-11-23","saturated":false},{"slug":"webarena-verified","family":"webarena-verified","name":"WebArena-Verified","nStates":12,"nModels":12,"difficulty":151.2,"saturation":176.2,"saturationExtrapolated":true,"frontierPct":81.9,"extrapolated":false,"extrapolation":null,"maxTested":161.2,"saturationDate":"2027-03-12","saturationDateLo":"2026-12-02","saturationDateHi":"2027-08-01","saturated":false},{"slug":"cybergym","family":"cybergym","name":"CyberGym","nStates":55,"nModels":43,"difficulty":151.3,"saturation":163.9,"saturationExtrapolated":false,"frontierPct":94.2,"extrapolated":false,"extrapolation":null,"maxTested":165.2,"saturationDate":"2026-06-09","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-01","saturated":true},{"slug":"healthbench-professional","family":"healthbench-professional","name":"HealthBench Professional","nStates":34,"nModels":30,"difficulty":151.3,"saturation":199.9,"saturationExtrapolated":true,"frontierPct":68.9,"extrapolated":false,"extrapolation":null,"maxTested":169.8,"saturationDate":"2028-08-23","saturationDateLo":"2028-04-18","saturationDateHi":"2029-03-03","saturated":false},{"slug":"surface-evolver-bench","family":"surface-evolver-bench","name":"Surface Evolver Bench","nStates":28,"nModels":26,"difficulty":151.3,"saturation":172.1,"saturationExtrapolated":true,"frontierPct":86.2,"extrapolated":false,"extrapolation":null,"maxTested":163.9,"saturationDate":"2026-11-29","saturationDateLo":"2026-09-02","saturationDateHi":"2027-03-25","saturated":false},{"slug":"usamo-2026","family":"usamo-2026","name":"USAMO 2026","nStates":19,"nModels":13,"difficulty":151.5,"saturation":159.7,"saturationExtrapolated":false,"frontierPct":99.0,"extrapolated":false,"extrapolation":null,"maxTested":164.6,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-04","saturationDateHi":"2026-03-05","saturated":true},{"slug":"biomysterybench-vals","family":"biomysterybench-vals","name":"BioMysteryBench (Vals Terminus 2)","nStates":27,"nModels":27,"difficulty":151.6,"saturation":178.5,"saturationExtrapolated":true,"frontierPct":78.7,"extrapolated":true,"extrapolation":"below","maxTested":167.1,"saturationDate":"2027-05-25","saturationDateLo":"2027-03-02","saturationDateHi":"2027-09-04","saturated":false},{"slug":"mrcr-v2-8-needle-context-arena-128k-auc","family":"mrcr-v2-8-needle-context-arena-128k-auc","name":"MRCR v2 (8-needle, Context Arena) - 128K AUC","nStates":178,"nModels":68,"difficulty":151.6,"saturation":172.2,"saturationExtrapolated":true,"frontierPct":85.4,"extrapolated":false,"extrapolation":null,"maxTested":165.1,"saturationDate":"2026-12-19","saturationDateLo":"2026-09-02","saturationDateHi":"2027-03-11","saturated":false},{"slug":"kernelbench-hard-normalized-h100","family":"kernelbench-hard-normalized-h100","name":"KernelBench Hard Normalized (H100)","nStates":13,"nModels":13,"difficulty":151.7,"saturation":190.6,"saturationExtrapolated":true,"frontierPct":80.2,"extrapolated":false,"extrapolation":null,"maxTested":164.6,"saturationDate":"2027-07-10","saturationDateLo":"2026-09-02","saturationDateHi":"2030-05-22","saturated":false},{"slug":"terminal-bench-1-0","family":"terminal-bench-1-0","name":"Terminal-Bench 1.0","nStates":17,"nModels":15,"difficulty":151.7,"saturation":186.3,"saturationExtrapolated":true,"frontierPct":77.4,"extrapolated":true,"extrapolation":"above","maxTested":149.8,"saturationDate":"2027-06-01","saturationDateLo":"2026-07-24","saturationDateHi":"2030-01-23","saturated":false},{"slug":"bridge-medical-zero-shot","family":"bridge-medical-zero-shot","name":"BRIDGE Medical (zero-shot)","nStates":72,"nModels":62,"difficulty":152.1,"saturation":253.8,"saturationExtrapolated":true,"frontierPct":58.7,"extrapolated":false,"extrapolation":null,"maxTested":160.0,"saturationDate":"2032-02-26","saturationDateLo":"2031-07-05","saturationDateHi":"2033-01-15","saturated":false},{"slug":"claw-eval-mm-pass-3","family":"claw-eval-mm-pass-3","name":"ClawEval-MM - pass@3","nStates":9,"nModels":9,"difficulty":152.2,"saturation":173.3,"saturationExtrapolated":true,"frontierPct":86.5,"extrapolated":false,"extrapolation":null,"maxTested":161.6,"saturationDate":"2026-11-23","saturationDateLo":"2026-09-02","saturationDateHi":"2027-07-15","saturated":false},{"slug":"llm-divergent-thinking-score","family":"llm-divergent-thinking-score","name":"LLM Divergent Thinking Score","nStates":17,"nModels":17,"difficulty":152.2,"saturation":338.7,"saturationExtrapolated":true,"frontierPct":55.1,"extrapolated":true,"extrapolation":"above","maxTested":138.2,"saturationDate":"2037-03-25","saturationDateLo":"2032-12-23","saturationDateHi":"2044-04-23","saturated":false},{"slug":"mindcube-tiny","family":"mindcube","name":"MindCube","nStates":58,"nModels":56,"difficulty":152.2,"saturation":174.4,"saturationExtrapolated":true,"frontierPct":90.9,"extrapolated":false,"extrapolation":null,"maxTested":170.6,"saturationDate":"2026-11-06","saturationDateLo":"2026-09-02","saturationDateHi":"2027-01-22","saturated":false},{"slug":"vulcanbench-frontier-v4","family":"vulcanbench-frontier-v4","name":"VulcanBench Frontier v4","nStates":55,"nModels":12,"difficulty":152.2,"saturation":166.7,"saturationExtrapolated":false,"frontierPct":91.7,"extrapolated":false,"extrapolation":null,"maxTested":167.8,"saturationDate":"2026-09-01","saturationDateLo":"2026-07-20","saturationDateHi":"2026-09-02","saturated":true},{"slug":"longbench-v2","family":"longbench-v2","name":"LongBench v2","nStates":55,"nModels":50,"difficulty":152.3,"saturation":194.8,"saturationExtrapolated":true,"frontierPct":76.3,"extrapolated":false,"extrapolation":null,"maxTested":163.0,"saturationDate":"2028-06-12","saturationDateLo":"2028-02-27","saturationDateHi":"2028-10-25","saturated":false},{"slug":"matplotgym","family":"matplotgym","name":"matplotgym","nStates":9,"nModels":9,"difficulty":152.5,"saturation":251.5,"saturationExtrapolated":true,"frontierPct":59.9,"extrapolated":false,"extrapolation":null,"maxTested":165.5,"saturationDate":"2031-07-18","saturationDateLo":"2029-09-22","saturationDateHi":"2035-06-27","saturated":false},{"slug":"multinrc-scale-ai-2","family":"multinrc-scale-ai-2","name":"MultiNRC","nStates":43,"nModels":37,"difficulty":152.5,"saturation":174.7,"saturationExtrapolated":true,"frontierPct":82.4,"extrapolated":false,"extrapolation":null,"maxTested":160.1,"saturationDate":"2027-02-14","saturationDateLo":"2026-12-06","saturationDateHi":"2027-05-24","saturated":false},{"slug":"legacy-bench-fireworks","family":"legacy-bench-fireworks","name":"Legacy-Bench (Fireworks, avg@3)","nStates":12,"nModels":12,"difficulty":152.6,"saturation":278.2,"saturationExtrapolated":true,"frontierPct":58.7,"extrapolated":true,"extrapolation":"below","maxTested":168.1,"saturationDate":"2032-10-24","saturationDateLo":"2029-12-01","saturationDateHi":"2041-09-05","saturated":false},{"slug":"mcpmark","family":"mcpmark","name":"MCPMark","nStates":23,"nModels":22,"difficulty":152.6,"saturation":168.2,"saturationExtrapolated":true,"frontierPct":89.8,"extrapolated":false,"extrapolation":null,"maxTested":156.1,"saturationDate":"2026-09-02","saturationDateLo":"2026-06-09","saturationDateHi":"2026-12-14","saturated":true},{"slug":"secprbench","family":"secprbench","name":"SecPRBench (recall)","nStates":8,"nModels":8,"difficulty":152.6,"saturation":186.4,"saturationExtrapolated":true,"frontierPct":79.5,"extrapolated":true,"extrapolation":"below","maxTested":167.4,"saturationDate":"2027-06-28","saturationDateLo":"2026-12-08","saturationDateHi":"2029-04-11","saturated":false},{"slug":"frontiermath-tiers-1-3-v2-private","family":"frontiermath-tiers-1-3-v2-private","name":"FrontierMath Tiers 1-3 (v2) - Private","nStates":120,"nModels":84,"difficulty":152.8,"saturation":168.7,"saturationExtrapolated":true,"frontierPct":89.4,"extrapolated":false,"extrapolation":null,"maxTested":168.9,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2026-09-21","saturated":true},{"slug":"simpleqa","family":"simpleqa","name":"SimpleQA","nStates":76,"nModels":69,"difficulty":153.0,"saturation":191.6,"saturationExtrapolated":true,"frontierPct":70.8,"extrapolated":true,"extrapolation":"above","maxTested":149.3,"saturationDate":"2028-02-22","saturationDateLo":"2027-07-27","saturationDateHi":"2028-10-05","saturated":false},{"slug":"unitybench-mini3d","family":"unitybench-mini3d","name":"UnityBench-Mini3D","nStates":13,"nModels":13,"difficulty":153.0,"saturation":205.4,"saturationExtrapolated":true,"frontierPct":71.5,"extrapolated":true,"extrapolation":"below","maxTested":165.1,"saturationDate":"2028-06-03","saturationDateLo":"2027-05-18","saturationDateHi":"2032-05-09","saturated":false},{"slug":"apex-agents-1-1","family":"apex-agents-1-1","name":"APEX-Agents 1.1","nStates":62,"nModels":58,"difficulty":153.2,"saturation":176.5,"saturationExtrapolated":true,"frontierPct":79.4,"extrapolated":false,"extrapolation":null,"maxTested":168.3,"saturationDate":"2027-04-13","saturationDateLo":"2027-01-29","saturationDateHi":"2027-06-14","saturated":false},{"slug":"bankertoolbench","family":"bankertoolbench","name":"BankerToolBench","nStates":9,"nModels":9,"difficulty":153.2,"saturation":183.5,"saturationExtrapolated":true,"frontierPct":75.9,"extrapolated":false,"extrapolation":null,"maxTested":157.4,"saturationDate":"2027-07-01","saturationDateLo":"2027-01-25","saturationDateHi":"2028-05-16","saturated":false},{"slug":"sci-agent-exam","family":"sci-agent-exam","name":"SCI-AGENT-EXAM","nStates":10,"nModels":10,"difficulty":153.2,"saturation":192.0,"saturationExtrapolated":true,"frontierPct":74.5,"extrapolated":true,"extrapolation":"below","maxTested":169.5,"saturationDate":"2027-10-21","saturationDateLo":"2027-04-13","saturationDateHi":"2028-10-25","saturated":false},{"slug":"vpct","family":"vpct","name":"VPCT","nStates":35,"nModels":25,"difficulty":153.3,"saturation":163.2,"saturationExtrapolated":false,"frontierPct":98.4,"extrapolated":false,"extrapolation":null,"maxTested":157.8,"saturationDate":"2026-02-11","saturationDateLo":"2026-02-11","saturationDateHi":"2026-06-09","saturated":true},{"slug":"aa-briefcase-v1-1","family":"aa-briefcase","name":"AA-Briefcase Elo","nStates":244,"nModels":153,"difficulty":153.4,"saturation":179.9,"saturationExtrapolated":true,"frontierPct":75.9,"extrapolated":false,"extrapolation":null,"maxTested":168.1},{"slug":"gpqa-openllm-harness","family":"gpqa-openllm-harness","name":"GPQA (Open LLM Leaderboard v2)","nStates":121,"nModels":121,"difficulty":153.4,"saturation":195.1,"saturationExtrapolated":true,"frontierPct":76.6,"extrapolated":true,"extrapolation":"above","maxTested":126.1,"saturationDate":"2028-05-20","saturationDateLo":"2027-10-22","saturationDateHi":"2029-01-08","saturated":false},{"slug":"missionreport","family":"missionreport","name":"MissionReport","nStates":9,"nModels":9,"difficulty":153.4,"saturation":206.2,"saturationExtrapolated":true,"frontierPct":67.0,"extrapolated":true,"extrapolation":"below","maxTested":167.2,"saturationDate":"2028-09-29","saturationDateLo":"2027-12-26","saturationDateHi":"2030-06-13","saturated":false},{"slug":"simplebench","family":"simplebench","name":"SimpleBench","nStates":102,"nModels":100,"difficulty":153.4,"saturation":178.6,"saturationExtrapolated":true,"frontierPct":81.4,"extrapolated":false,"extrapolation":null,"maxTested":168.7,"saturationDate":"2027-05-28","saturationDateLo":"2027-03-17","saturationDateHi":"2027-08-05","saturated":false},{"slug":"hardgeobench-country-accuracy","family":"hardgeobench-avg-distance-miles","name":"HardGeoBench Avg Distance","nStates":89,"nModels":70,"difficulty":153.5,"saturation":303.7,"saturationExtrapolated":true,"frontierPct":55.8,"extrapolated":false,"extrapolation":null,"maxTested":168.1,"saturationDate":"2035-01-14","saturationDateLo":"2032-04-26","saturationDateHi":"2039-11-20","saturated":false},{"slug":"webdev-arena","family":"webdev-arena","name":"LMArena Code Arena WebDev","nStates":125,"nModels":111,"difficulty":153.5,"saturation":173.6,"saturationExtrapolated":true,"frontierPct":81.6,"extrapolated":false,"extrapolation":null,"maxTested":167.7},{"slug":"slopcodebench-core-solved-scbench","family":"slopcodebench-core-solved-scbench","name":"SlopCodeBench - Core Solved","nStates":16,"nModels":14,"difficulty":153.6,"saturation":172.3,"saturationExtrapolated":true,"frontierPct":84.7,"extrapolated":false,"extrapolation":null,"maxTested":159.8,"saturationDate":"2026-12-09","saturationDateLo":"2026-09-02","saturationDateHi":"2027-05-05","saturated":false},{"slug":"erqa","family":"erqa","name":"ERQA","nStates":111,"nModels":93,"difficulty":153.7,"saturation":195.0,"saturationExtrapolated":true,"frontierPct":77.9,"extrapolated":false,"extrapolation":null,"maxTested":169.8,"saturationDate":"2028-04-12","saturationDateLo":"2027-12-31","saturationDateHi":"2028-07-14","saturated":false},{"slug":"global-piqa-nonparallel-loglikelihood-accuracy","family":"global-piqa-nonparallel-loglikelihood-accuracy","name":"Global PIQA - Non-Parallel (Log-Likelihood Accuracy)","nStates":20,"nModels":20,"difficulty":153.9,"saturation":213.6,"saturationExtrapolated":true,"frontierPct":82.0,"extrapolated":true,"extrapolation":"above","maxTested":122.3,"saturationDate":"2029-06-17","saturationDateLo":"2027-05-16","saturationDateHi":"2032-01-27","saturated":false},{"slug":"llm-debate-benchmark","family":"llm-debate-benchmark","name":"LLM Debate Benchmark","nStates":53,"nModels":50,"difficulty":153.9,"saturation":174.0,"saturationExtrapolated":true,"frontierPct":81.0,"extrapolated":false,"extrapolation":null,"maxTested":167.2},{"slug":"medxpertqa-text","family":"medxpertqa-text","name":"MedXpertQA Text","nStates":14,"nModels":14,"difficulty":153.9,"saturation":186.4,"saturationExtrapolated":true,"frontierPct":75.8,"extrapolated":false,"extrapolation":null,"maxTested":159.6,"saturationDate":"2027-10-14","saturationDateLo":"2027-05-20","saturationDateHi":"2028-04-27","saturated":false},{"slug":"osworld-2-1-full-reported-partial","family":"osworld-2-1-full-500-binary","name":"OSWorld 2.1 Full, 500-step Binary Accuracy","nStates":9,"nModels":8,"difficulty":153.9,"saturation":175.8,"saturationExtrapolated":true,"frontierPct":82.2,"extrapolated":true,"extrapolation":"below","maxTested":168.2,"saturationDate":"2027-02-27","saturationDateLo":"2026-12-17","saturationDateHi":"2027-08-14","saturated":false},{"slug":"public-benefits-bench-multi-turn","family":"public-benefits-bench-multi-turn","name":"Public Benefits Bench v1 - Multi Turn","nStates":13,"nModels":13,"difficulty":153.9,"saturation":205.6,"saturationExtrapolated":true,"frontierPct":64.9,"extrapolated":false,"extrapolation":null,"maxTested":164.2,"saturationDate":"2029-02-07","saturationDateLo":"2028-07-22","saturationDateHi":"2029-10-13","saturated":false},{"slug":"public-benefits-bench-web-search","family":"public-benefits-bench-web-search","name":"Public Benefits Bench v1 - Web Search","nStates":13,"nModels":13,"difficulty":154.0,"saturation":200.5,"saturationExtrapolated":true,"frontierPct":65.9,"extrapolated":false,"extrapolation":null,"maxTested":164.0,"saturationDate":"2028-08-23","saturationDateLo":"2028-01-26","saturationDateHi":"2029-12-07","saturated":false},{"slug":"superchem","family":"superchem","name":"SUPERChem","nStates":32,"nModels":29,"difficulty":154.1,"saturation":186.8,"saturationExtrapolated":true,"frontierPct":74.7,"extrapolated":false,"extrapolation":null,"maxTested":170.4,"saturationDate":"2027-09-16","saturationDateLo":"2027-04-14","saturationDateHi":"2028-06-10","saturated":false},{"slug":"aa-lcr-mercor-extended","family":"aa-lcr-mercor-extended","name":"AA-LCR Extended (Mercor held-out)","nStates":47,"nModels":46,"difficulty":154.2,"saturation":179.2,"saturationExtrapolated":true,"frontierPct":77.1,"extrapolated":false,"extrapolation":null,"maxTested":168.5,"saturationDate":"2027-06-03","saturationDateLo":"2027-03-22","saturationDateHi":"2027-08-23","saturated":false},{"slug":"eq-bench-4","family":"eq-bench-4","name":"EQ-Bench 4","nStates":31,"nModels":31,"difficulty":154.2,"saturation":174.2,"saturationExtrapolated":true,"frontierPct":81.2,"extrapolated":false,"extrapolation":null,"maxTested":167.2},{"slug":"hemingway-bench-surge-ai-2","family":"hemingway-bench-surge-ai-2","name":"Hemingway-bench","nStates":40,"nModels":40,"difficulty":154.2,"saturation":175.5,"saturationExtrapolated":true,"frontierPct":81.1,"extrapolated":false,"extrapolation":null,"maxTested":165.6},{"slug":"minebench-bradley-terry-rating","family":"minebench-bradley-terry-rating","name":"MineBench \u2014 Bradley-Terry Rating","nStates":72,"nModels":72,"difficulty":154.3,"saturation":174.4,"saturationExtrapolated":true,"frontierPct":80.8,"extrapolated":false,"extrapolation":null,"maxTested":169.7},{"slug":"arxivmath-overall-matharena","family":"arxivmath-overall-matharena","name":"ArXivMath \u2014 Overall","nStates":12,"nModels":12,"difficulty":154.4,"saturation":177.5,"saturationExtrapolated":true,"frontierPct":80.0,"extrapolated":false,"extrapolation":null,"maxTested":164.4,"saturationDate":"2027-03-19","saturationDateLo":"2026-12-23","saturationDateHi":"2027-07-15","saturated":false},{"slug":"balrog","family":"balrog","name":"BALROG","nStates":39,"nModels":37,"difficulty":154.4,"saturation":204.4,"saturationExtrapolated":true,"frontierPct":64.8,"extrapolated":false,"extrapolation":null,"maxTested":168.6,"saturationDate":"2029-01-20","saturationDateLo":"2028-08-16","saturationDateHi":"2029-08-06","saturated":false},{"slug":"blenderworld","family":"blenderworld","name":"BlenderWorld","nStates":10,"nModels":10,"difficulty":154.6,"saturation":197.4,"saturationExtrapolated":true,"frontierPct":72.6,"extrapolated":true,"extrapolation":"below","maxTested":168.8,"saturationDate":"2027-12-30","saturationDateLo":"2027-04-14","saturationDateHi":"2030-10-20","saturated":false},{"slug":"audio-multichallenge-text-output","family":"audio-multichallenge-text-output","name":"Audio MultiChallenge (Text Output)","nStates":17,"nModels":16,"difficulty":154.7,"saturation":207.7,"saturationExtrapolated":true,"frontierPct":65.5,"extrapolated":false,"extrapolation":null,"maxTested":168.8,"saturationDate":"2029-01-28","saturationDateLo":"2028-04-30","saturationDateHi":"2030-01-07","saturated":false},{"slug":"babaisbench-lake","family":"babaisbench-lake","name":"BabaIsBench Stage 1: The Lake","nStates":19,"nModels":19,"difficulty":154.7,"saturation":163.8,"saturationExtrapolated":false,"frontierPct":97.2,"extrapolated":true,"extrapolation":"below","maxTested":166.1,"saturationDate":"2026-04-07","saturationDateLo":"2026-02-11","saturationDateHi":"2026-09-01","saturated":true},{"slug":"imo-proofbench-advanced-usamo-2025","family":"imo-proofbench-advanced-usamo-2025","name":"IMO-ProofBench Advanced: USAMO 2025","nStates":22,"nModels":22,"difficulty":154.7,"saturation":169.1,"saturationExtrapolated":true,"frontierPct":89.9,"extrapolated":false,"extrapolation":null,"maxTested":164.2,"saturationDate":"2026-09-02","saturationDateLo":"2026-04-07","saturationDateHi":"2026-12-27","saturated":true},{"slug":"cursorbench-3-2","family":"cursorbench-3-2","name":"CursorBench 3.2","nStates":58,"nModels":17,"difficulty":154.9,"saturation":182.1,"saturationExtrapolated":true,"frontierPct":74.4,"extrapolated":false,"extrapolation":null,"maxTested":166.8,"saturationDate":"2027-08-08","saturationDateLo":"2027-06-02","saturationDateHi":"2027-10-14","saturated":false},{"slug":"officeqa-pro-card","family":"officeqa-pro-card","name":"OfficeQA Pro","nStates":13,"nModels":12,"difficulty":154.9,"saturation":203.6,"saturationExtrapolated":true,"frontierPct":70.8,"extrapolated":true,"extrapolation":"below","maxTested":165.6,"saturationDate":"2028-04-19","saturationDateLo":"2027-03-12","saturationDateHi":"2031-09-26","saturated":false},{"slug":"scicode","family":"scicode","name":"SciCode","nStates":43,"nModels":39,"difficulty":155.0,"saturation":202.0,"saturationExtrapolated":true,"frontierPct":65.2,"extrapolated":false,"extrapolation":null,"maxTested":159.6,"saturationDate":"2028-11-16","saturationDateLo":"2028-07-25","saturationDateHi":"2029-03-08","saturated":false},{"slug":"spatialviz-bench","family":"spatialviz-bench","name":"SpatialViz-Bench","nStates":50,"nModels":50,"difficulty":155.0,"saturation":173.0,"saturationExtrapolated":true,"frontierPct":88.3,"extrapolated":false,"extrapolation":null,"maxTested":168.4,"saturationDate":"2026-12-13","saturationDateLo":"2026-09-02","saturationDateHi":"2027-02-12","saturated":false},{"slug":"vibe-code","family":"vibe-code","name":"Vibe Code Bench v1.1","nStates":114,"nModels":109,"difficulty":155.1,"saturation":165.0,"saturationExtrapolated":false,"frontierPct":93.6,"extrapolated":false,"extrapolation":null,"maxTested":167.6,"saturationDate":"2026-07-24","saturationDateLo":"2026-06-09","saturationDateHi":"2026-09-01","saturated":true},{"slug":"tax-agent-bench","family":"tax-agent-bench","name":"Tax Agent Bench","nStates":72,"nModels":71,"difficulty":155.2,"saturation":177.7,"saturationExtrapolated":true,"frontierPct":76.3,"extrapolated":false,"extrapolation":null,"maxTested":167.5,"saturationDate":"2027-05-25","saturationDateLo":"2027-03-20","saturationDateHi":"2027-07-23","saturated":false},{"slug":"terminal-bench-hard-artificial-analysis","family":"terminal-bench-hard-artificial-analysis","name":"Terminal-Bench Hard","nStates":421,"nModels":304,"difficulty":155.2,"saturation":175.9,"saturationExtrapolated":true,"frontierPct":78.1,"extrapolated":false,"extrapolation":null,"maxTested":162.7,"saturationDate":"2027-04-07","saturationDateLo":"2027-02-01","saturationDateHi":"2027-05-19","saturated":false},{"slug":"emb","family":"emb","name":"Excel Modeling Benchmark","nStates":74,"nModels":73,"difficulty":155.3,"saturation":175.9,"saturationExtrapolated":true,"frontierPct":78.1,"extrapolated":false,"extrapolation":null,"maxTested":167.9,"saturationDate":"2027-04-04","saturationDateLo":"2027-02-06","saturationDateHi":"2027-06-06","saturated":false},{"slug":"latchbio-singlecellbench-card","family":"latchbio-singlecellbench-card","name":"scBench","nStates":21,"nModels":19,"difficulty":155.4,"saturation":189.5,"saturationExtrapolated":true,"frontierPct":69.6,"extrapolated":false,"extrapolation":null,"maxTested":165.2,"saturationDate":"2028-01-18","saturationDateLo":"2027-09-26","saturationDateHi":"2028-06-22","saturated":false},{"slug":"studentbench-lesson-planning","family":"studentbench-lesson-planning","name":"ATLAS-Education (StudentBench) Lesson Planning","nStates":13,"nModels":12,"difficulty":155.4,"saturation":189.4,"saturationExtrapolated":true,"frontierPct":73.8,"extrapolated":false,"extrapolation":null,"maxTested":164.5},{"slug":"aurora-multilingual-terminal-bench","family":"aurora-multilingual-terminal-bench","name":"AURORA Multilingual Terminal-Bench","nStates":10,"nModels":10,"difficulty":155.6,"saturation":177.6,"saturationExtrapolated":true,"frontierPct":79.0,"extrapolated":false,"extrapolation":null,"maxTested":167.4,"saturationDate":"2027-03-25","saturationDateLo":"2026-11-29","saturationDateHi":"2027-07-28","saturated":false},{"slug":"vita-bench","family":"vita-bench","name":"VitaBench","nStates":27,"nModels":25,"difficulty":155.7,"saturation":190.3,"saturationExtrapolated":true,"frontierPct":69.1,"extrapolated":false,"extrapolation":null,"maxTested":162.2,"saturationDate":"2028-02-02","saturationDateLo":"2027-08-03","saturationDateHi":"2028-10-14","saturated":false},{"slug":"skillsbench-with-skills","family":"skillsbench-with-skills","name":"SkillsBench with Skills","nStates":53,"nModels":47,"difficulty":155.8,"saturation":189.5,"saturationExtrapolated":true,"frontierPct":68.3,"extrapolated":false,"extrapolation":null,"maxTested":163.3,"saturationDate":"2028-02-05","saturationDateLo":"2027-10-04","saturationDateHi":"2028-07-08","saturated":false},{"slug":"arena-ai-document","family":"arena-ai-document","name":"Document Arena","nStates":41,"nModels":35,"difficulty":156.0,"saturation":175.7,"saturationExtrapolated":true,"frontierPct":78.8,"extrapolated":false,"extrapolation":null,"maxTested":167.4},{"slug":"llm-creative-story-writing","family":"llm-creative-story-writing","name":"LLM Creative Story Writing","nStates":54,"nModels":50,"difficulty":156.0,"saturation":184.9,"saturationExtrapolated":true,"frontierPct":72.5,"extrapolated":false,"extrapolation":null,"maxTested":168.0},{"slug":"seal-0","family":"seal-0","name":"Seal-0","nStates":35,"nModels":26,"difficulty":156.1,"saturation":179.3,"saturationExtrapolated":true,"frontierPct":76.7,"extrapolated":true,"extrapolation":"above","maxTested":152.7,"saturationDate":"2027-05-09","saturationDateLo":"2026-09-02","saturationDateHi":"2028-04-14","saturated":false},{"slug":"spatialbench-benchmarks-bio","family":"spatialbench-benchmarks-bio","name":"SpatialBench","nStates":17,"nModels":16,"difficulty":156.1,"saturation":194.4,"saturationExtrapolated":true,"frontierPct":66.8,"extrapolated":false,"extrapolation":null,"maxTested":160.0,"saturationDate":"2028-05-12","saturationDateLo":"2027-12-07","saturationDateHi":"2029-01-16","saturated":false},{"slug":"voxelbench-image-rating","family":"voxelbench-image-rating","name":"VoxelBench Image Rating","nStates":10,"nModels":10,"difficulty":156.1,"saturation":183.6,"saturationExtrapolated":true,"frontierPct":77.3,"extrapolated":false,"extrapolation":null,"maxTested":168.8},{"slug":"refspatial-bench","family":"refspatial-bench","name":"RefSpatial-Bench","nStates":25,"nModels":22,"difficulty":156.2,"saturation":247.0,"saturationExtrapolated":true,"frontierPct":58.4,"extrapolated":true,"extrapolation":"above","maxTested":152.7,"saturationDate":"2031-04-26","saturationDateLo":"2028-11-07","saturationDateHi":"2035-09-06","saturated":false},{"slug":"perceptionbench","family":"perceptionbench","name":"PerceptionBench","nStates":20,"nModels":17,"difficulty":156.3,"saturation":217.5,"saturationExtrapolated":true,"frontierPct":61.4,"extrapolated":false,"extrapolation":null,"maxTested":165.8,"saturationDate":"2029-08-22","saturationDateLo":"2028-08-06","saturationDateHi":"2031-03-27","saturated":false},{"slug":"scicode-1-0-1-artificial-analysis","family":"scicode-1-0-1-artificial-analysis","name":"SciCode 1.0.1 (Artificial Analysis)","nStates":234,"nModels":150,"difficulty":156.3,"saturation":214.2,"saturationExtrapolated":true,"frontierPct":61.1,"extrapolated":false,"extrapolation":null,"maxTested":169.0,"saturationDate":"2029-09-11","saturationDateLo":"2029-06-05","saturationDateHi":"2029-12-22","saturated":false},{"slug":"cvdp-517","family":"cvdp-517","name":"CVDP (517-problem set)","nStates":24,"nModels":22,"difficulty":156.4,"saturation":215.4,"saturationExtrapolated":true,"frontierPct":60.3,"extrapolated":false,"extrapolation":null,"maxTested":167.9,"saturationDate":"2029-08-16","saturationDateLo":"2028-12-28","saturationDateHi":"2030-09-25","saturated":false},{"slug":"cwe-bench-v1","family":"cwe-bench-v1","name":"CWE-bench v1 Programmatic Pass@1","nStates":12,"nModels":12,"difficulty":156.4,"saturation":193.1,"saturationExtrapolated":true,"frontierPct":67.9,"extrapolated":false,"extrapolation":null,"maxTested":168.8,"saturationDate":"2028-03-06","saturationDateLo":"2027-10-01","saturationDateHi":"2028-12-07","saturated":false},{"slug":"hebbia-financial-services-benchmark","family":"hebbia-financial-services-benchmark","name":"Hebbia Financial Services Benchmark","nStates":10,"nModels":10,"difficulty":156.5,"saturation":187.4,"saturationExtrapolated":true,"frontierPct":72.9,"extrapolated":true,"extrapolation":"below","maxTested":169.2,"saturationDate":"2027-09-04","saturationDateLo":"2027-03-31","saturationDateHi":"2028-10-15","saturated":false},{"slug":"ioi","family":"ioi","name":"IOI","nStates":58,"nModels":53,"difficulty":156.5,"saturation":168.4,"saturationExtrapolated":true,"frontierPct":88.0,"extrapolated":false,"extrapolation":null,"maxTested":164.9,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2026-11-30","saturated":true},{"slug":"long-horizon-terminal-bench-2-0","family":"long-horizon-terminal-bench-2-0","name":"Long-Horizon Terminal-Bench 2.0 (mean reward)","nStates":12,"nModels":12,"difficulty":156.5,"saturation":205.4,"saturationExtrapolated":true,"frontierPct":65.3,"extrapolated":false,"extrapolation":null,"maxTested":168.1,"saturationDate":"2028-08-19","saturationDateLo":"2027-11-02","saturationDateHi":"2030-05-30","saturated":false},{"slug":"medxpertqa-mm-mercor-extended","family":"medxpertqa-mm-mercor-extended","name":"MedXpertQA MM Extended (Mercor held-out)","nStates":40,"nModels":39,"difficulty":156.8,"saturation":212.4,"saturationExtrapolated":true,"frontierPct":64.6,"extrapolated":false,"extrapolation":null,"maxTested":173.8,"saturationDate":"2029-03-07","saturationDateLo":"2028-08-01","saturationDateHi":"2030-01-29","saturated":false},{"slug":"nl2repo-reported","family":"nl2repo-reported","name":"NL2Repo - release-reported repository generation score","nStates":34,"nModels":29,"difficulty":156.8,"saturation":177.9,"saturationExtrapolated":true,"frontierPct":74.8,"extrapolated":false,"extrapolation":null,"maxTested":165.0,"saturationDate":"2027-05-18","saturationDateLo":"2027-03-07","saturationDateHi":"2027-07-29","saturated":false},{"slug":"tampered-ml","family":"tampered-ml","name":"Tampered ML","nStates":22,"nModels":22,"difficulty":156.8,"saturation":181.6,"saturationExtrapolated":true,"frontierPct":80.5,"extrapolated":false,"extrapolation":null,"maxTested":165.4,"saturationDate":"2027-07-08","saturationDateLo":"2027-03-11","saturationDateHi":"2027-12-01","saturated":false},{"slug":"toolathlon","family":"toolathlon","name":"Toolathlon","nStates":60,"nModels":41,"difficulty":156.8,"saturation":179.2,"saturationExtrapolated":true,"frontierPct":74.2,"extrapolated":false,"extrapolation":null,"maxTested":164.1,"saturationDate":"2027-06-08","saturationDateLo":"2027-03-16","saturationDateHi":"2027-09-14","saturated":false},{"slug":"voratiq-coding-agent-leaderboard","family":"voratiq-coding-agent-leaderboard","name":"Voratiq Coding Agent Leaderboard","nStates":26,"nModels":19,"difficulty":156.8,"saturation":175.6,"saturationExtrapolated":true,"frontierPct":81.3,"extrapolated":false,"extrapolation":null,"maxTested":165.0},{"slug":"hle-with-tools","family":"hle-with-tools","name":"Humanity's Last Exam (with tools)","nStates":112,"nModels":73,"difficulty":156.9,"saturation":189.2,"saturationExtrapolated":true,"frontierPct":68.1,"extrapolated":false,"extrapolation":null,"maxTested":167.9,"saturationDate":"2028-02-16","saturationDateLo":"2027-11-21","saturationDateHi":"2028-04-30","saturated":false},{"slug":"deep-research-bench","family":"deep-research-bench","name":"Deep Research Bench","nStates":37,"nModels":22,"difficulty":157.0,"saturation":294.4,"saturationExtrapolated":true,"frontierPct":54.9,"extrapolated":false,"extrapolation":null,"maxTested":160.3,"saturationDate":"2034-04-26","saturationDateLo":"2031-10-14","saturationDateHi":"2038-06-01","saturated":false},{"slug":"lmarena-image-to-webdev-arena","family":"lmarena-image-to-webdev-arena","name":"Arena Image-to-WebDev","nStates":54,"nModels":48,"difficulty":157.0,"saturation":170.3,"saturationExtrapolated":true,"frontierPct":84.4,"extrapolated":false,"extrapolation":null,"maxTested":166.7},{"slug":"arc-agi-2","family":"arc-agi-2","name":"ARC-AGI-2","nStates":280,"nModels":93,"difficulty":157.2,"saturation":167.0,"saturationExtrapolated":false,"frontierPct":94.2,"extrapolated":false,"extrapolation":null,"maxTested":171.1,"saturationDate":"2026-06-09","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-01","saturated":true},{"slug":"buseybench-svg","family":"buseybench-svg","name":"BuseyBench SVG","nStates":252,"nModels":249,"difficulty":157.2,"saturation":201.2,"saturationExtrapolated":true,"frontierPct":63.4,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2028-11-04","saturationDateLo":"2028-07-29","saturationDateHi":"2029-02-09","saturated":false},{"slug":"scbench-v2","family":"scbench-v2","name":"ScBench V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":157.2,"saturation":207.9,"saturationExtrapolated":true,"frontierPct":63.3,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2028-11-22","saturationDateLo":"2028-02-01","saturationDateHi":"2030-06-27","saturated":false},{"slug":"deepplanning","family":"deepplanning","name":"DeepPlanning","nStates":23,"nModels":23,"difficulty":157.4,"saturation":176.0,"saturationExtrapolated":true,"frontierPct":77.5,"extrapolated":true,"extrapolation":"above","maxTested":156.0,"saturationDate":"2027-03-12","saturationDateLo":"2026-11-03","saturationDateHi":"2027-09-10","saturated":false},{"slug":"finance-agent-v2","family":"finance-agent-v2","name":"Finance Agent v2","nStates":83,"nModels":75,"difficulty":157.4,"saturation":192.8,"saturationExtrapolated":true,"frontierPct":64.7,"extrapolated":false,"extrapolation":null,"maxTested":167.7,"saturationDate":"2028-05-17","saturationDateLo":"2028-02-14","saturationDateHi":"2028-08-14","saturated":false},{"slug":"mle-bench-partial-30-average-position","family":"mle-bench-partial-30-average-position","name":"MLE-Bench Partial 30 - Average Position Score","nStates":8,"nModels":8,"difficulty":157.4,"saturation":197.3,"saturationExtrapolated":true,"frontierPct":67.4,"extrapolated":false,"extrapolation":null,"maxTested":159.4,"saturationDate":"2028-01-24","saturationDateLo":"2027-04-01","saturationDateHi":"2030-03-03","saturated":false},{"slug":"pencil-puzzle-bench-agentic-ppbench","family":"pencil-puzzle-bench-agentic-ppbench","name":"Pencil Puzzle Bench - Agentic","nStates":65,"nModels":49,"difficulty":157.6,"saturation":164.4,"saturationExtrapolated":false,"frontierPct":96.3,"extrapolated":false,"extrapolation":null,"maxTested":163.9,"saturationDate":"2026-06-09","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-01","saturated":true},{"slug":"pmc-vqa","family":"pmc-vqa","name":"PMC-VQA","nStates":25,"nModels":23,"difficulty":157.6,"saturation":254.3,"saturationExtrapolated":true,"frontierPct":67.4,"extrapolated":false,"extrapolation":null,"maxTested":161.6,"saturationDate":"2031-11-12","saturationDateLo":"2030-03-07","saturationDateHi":"2034-08-24","saturated":false},{"slug":"txbench-pp-benchmarks-bio","family":"txbench-pp-benchmarks-bio","name":"TxBench-PP","nStates":11,"nModels":11,"difficulty":157.6,"saturation":171.8,"saturationExtrapolated":true,"frontierPct":81.9,"extrapolated":false,"extrapolation":null,"maxTested":160.0,"saturationDate":"2026-12-19","saturationDateLo":"2026-09-02","saturationDateHi":"2027-03-07","saturated":false},{"slug":"sec-bench-pro-reported","family":"sec-bench-pro-caisi-183","name":"SEC-Bench Pro (CAISI 183 tasks)","nStates":11,"nModels":10,"difficulty":157.7,"saturation":169.7,"saturationExtrapolated":true,"frontierPct":87.4,"extrapolated":true,"extrapolation":"below","maxTested":168.0,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2027-01-12","saturated":true},{"slug":"benchcad-reported-iou","family":"benchcad-reported-iou","name":"BenchCAD - geometric overlap","nStates":15,"nModels":12,"difficulty":157.8,"saturation":192.7,"saturationExtrapolated":true,"frontierPct":74.0,"extrapolated":false,"extrapolation":null,"maxTested":171.1,"saturationDate":"2027-08-27","saturationDateLo":"2027-01-15","saturationDateHi":"2030-02-11","saturated":false},{"slug":"gamedevbench","family":"gamedevbench","name":"GameDevBench","nStates":21,"nModels":18,"difficulty":158.2,"saturation":183.7,"saturationExtrapolated":true,"frontierPct":71.1,"extrapolated":false,"extrapolation":null,"maxTested":168.3,"saturationDate":"2027-08-19","saturationDateLo":"2027-04-11","saturationDateHi":"2028-02-21","saturated":false},{"slug":"gbench-one-shot-coding-gscore","family":"gbench-one-shot-coding-gscore","name":"GBENCH One-shot Coding GScore","nStates":101,"nModels":101,"difficulty":158.2,"saturation":181.9,"saturationExtrapolated":true,"frontierPct":73.1,"extrapolated":false,"extrapolation":null,"maxTested":168.8,"saturationDate":"2027-07-09","saturationDateLo":"2027-04-20","saturationDateHi":"2027-09-14","saturated":false},{"slug":"lmca","family":"lmca","name":"LMCA","nStates":191,"nModels":135,"difficulty":158.2,"saturation":193.6,"saturationExtrapolated":true,"frontierPct":63.6,"extrapolated":false,"extrapolation":null,"maxTested":167.9,"saturationDate":"2028-05-31","saturationDateLo":"2028-03-23","saturationDateHi":"2028-08-06","saturated":false},{"slug":"lmarena-search-arena","family":"lmarena-search-arena","name":"Search Arena","nStates":17,"nModels":17,"difficulty":158.3,"saturation":208.6,"saturationExtrapolated":true,"frontierPct":62.8,"extrapolated":false,"extrapolation":null,"maxTested":168.4},{"slug":"ma-bench","family":"ma-bench","name":"MA-Bench","nStates":17,"nModels":17,"difficulty":158.3,"saturation":254.8,"saturationExtrapolated":true,"frontierPct":56.1,"extrapolated":false,"extrapolation":null,"maxTested":166.3,"saturationDate":"2031-12-13","saturationDateLo":"2030-02-07","saturationDateHi":"2035-09-12","saturated":false},{"slug":"apex-accounting-1-1","family":"apex-accounting-1-1","name":"APEX-Accounting 1.1","nStates":43,"nModels":43,"difficulty":158.4,"saturation":181.8,"saturationExtrapolated":true,"frontierPct":70.0,"extrapolated":false,"extrapolation":null,"maxTested":168.2,"saturationDate":"2027-08-12","saturationDateLo":"2027-05-06","saturationDateHi":"2027-11-12","saturated":false},{"slug":"graphwalks-bfs-1m","family":"graphwalks-bfs-1m","name":"GraphWalks BFS - 1M","nStates":13,"nModels":10,"difficulty":158.5,"saturation":182.5,"saturationExtrapolated":true,"frontierPct":82.3,"extrapolated":false,"extrapolation":null,"maxTested":164.0,"saturationDate":"2026-12-22","saturationDateLo":"2026-09-01","saturationDateHi":"2029-04-03","saturated":false},{"slug":"swe-bench-verified-mercor-extended","family":"swe-bench-verified-mercor-extended","name":"SWE-bench Verified Extended (Mercor held-out)","nStates":42,"nModels":41,"difficulty":158.6,"saturation":178.2,"saturationExtrapolated":true,"frontierPct":73.1,"extrapolated":false,"extrapolation":null,"maxTested":167.9,"saturationDate":"2027-05-16","saturationDateLo":"2027-02-12","saturationDateHi":"2027-10-01","saturated":false},{"slug":"swe-clar-implementation","family":"swe-clar-implementation","name":"SWE-Clar (implementation)","nStates":8,"nModels":8,"difficulty":158.6,"saturation":197.1,"saturationExtrapolated":true,"frontierPct":67.3,"extrapolated":false,"extrapolation":null,"maxTested":166.1,"saturationDate":"2027-12-20","saturationDateLo":"2027-05-09","saturationDateHi":"2030-10-03","saturated":false},{"slug":"frontiercode-1-1-extended-score-cognition","family":"frontiercode-1-1-extended-score-cognition","name":"FrontierCode 1.1 Extended - Score","nStates":132,"nModels":39,"difficulty":158.7,"saturation":187.4,"saturationExtrapolated":true,"frontierPct":66.4,"extrapolated":false,"extrapolation":null,"maxTested":168.4,"saturationDate":"2027-12-27","saturationDateLo":"2027-10-13","saturationDateHi":"2028-02-25","saturated":false},{"slug":"gdpval-aa-v2-normalized-artificial-analysis","family":"gdpval-aa-v2-normalized-artificial-analysis","name":"GDPval-AA v2 (normalized)","nStates":178,"nModels":134,"difficulty":158.7,"saturation":177.3,"saturationExtrapolated":true,"frontierPct":73.2,"extrapolated":false,"extrapolation":null,"maxTested":164.9,"saturationDate":"2027-05-04","saturationDateLo":"2027-03-04","saturationDateHi":"2027-06-25","saturated":false},{"slug":"taxcalcbench-ty24","family":"taxcalcbench-ty24","name":"TaxCalcBench TY24","nStates":17,"nModels":17,"difficulty":158.7,"saturation":190.4,"saturationExtrapolated":true,"frontierPct":66.4,"extrapolated":false,"extrapolation":null,"maxTested":161.9,"saturationDate":"2028-01-09","saturationDateLo":"2027-07-05","saturationDateHi":"2028-12-28","saturated":false},{"slug":"deepswe-datacurve-2","family":"deepswe-datacurve-2","name":"DeepSWE v1","nStates":29,"nModels":21,"difficulty":158.9,"saturation":167.1,"saturationExtrapolated":false,"frontierPct":90.6,"extrapolated":false,"extrapolation":null,"maxTested":160.5,"saturationDate":"2026-09-02","saturationDateLo":"2026-08-28","saturationDateHi":"2026-09-21","saturated":true},{"slug":"proof_bench","family":"proof_bench","name":"ProofBench","nStates":52,"nModels":51,"difficulty":158.9,"saturation":169.9,"saturationExtrapolated":true,"frontierPct":84.6,"extrapolated":false,"extrapolation":null,"maxTested":165.0,"saturationDate":"2026-11-04","saturationDateLo":"2026-09-02","saturationDateHi":"2026-12-28","saturated":false},{"slug":"critpt-v2","family":"critpt-v2","name":"CritPt v2 (agentic, avg@5)","nStates":98,"nModels":50,"difficulty":159.1,"saturation":173.0,"saturationExtrapolated":true,"frontierPct":81.1,"extrapolated":false,"extrapolation":null,"maxTested":169.2,"saturationDate":"2026-12-31","saturationDateLo":"2026-11-04","saturationDateHi":"2027-02-05","saturated":false},{"slug":"deepswe-1-1","family":"deepswe-1-1","name":"DeepSWE v1.1","nStates":150,"nModels":72,"difficulty":159.1,"saturation":170.1,"saturationExtrapolated":true,"frontierPct":85.5,"extrapolated":false,"extrapolation":null,"maxTested":168.6,"saturationDate":"2026-11-06","saturationDateLo":"2026-09-02","saturationDateHi":"2026-12-09","saturated":false},{"slug":"proofbench-v1-1","family":"proofbench-v1-1","name":"ProofBench v1.1","nStates":51,"nModels":50,"difficulty":159.1,"saturation":166.7,"saturationExtrapolated":false,"frontierPct":94.1,"extrapolated":false,"extrapolation":null,"maxTested":169.6,"saturationDate":"2026-09-01","saturationDateLo":"2026-04-07","saturationDateHi":"2026-09-02","saturated":true},{"slug":"pwnbench-v0-1","family":"pwnbench-v0-1","name":"PWNBench v0.1","nStates":13,"nModels":13,"difficulty":159.1,"saturation":199.5,"saturationExtrapolated":true,"frontierPct":62.5,"extrapolated":false,"extrapolation":null,"maxTested":164.9,"saturationDate":"2028-06-04","saturationDateLo":"2027-09-16","saturationDateHi":"2030-01-17","saturated":false},{"slug":"voxelbench-text-rating","family":"voxelbench-text-rating","name":"VoxelBench Text Rating","nStates":51,"nModels":51,"difficulty":159.1,"saturation":172.6,"saturationExtrapolated":true,"frontierPct":82.1,"extrapolated":false,"extrapolation":null,"maxTested":169.3},{"slug":"apex-matharena-2","family":"apex-matharena-2","name":"Apex (MathArena)","nStates":54,"nModels":49,"difficulty":159.3,"saturation":168.2,"saturationExtrapolated":false,"frontierPct":91.1,"extrapolated":false,"extrapolation":null,"maxTested":162.6,"saturationDate":"2026-09-02","saturationDateLo":"2026-07-05","saturationDateHi":"2026-09-02","saturated":true},{"slug":"professional-reasoning-benchmark-finance-scale-ai-2","family":"professional-reasoning-benchmark-finance-scale-ai-2","name":"Professional Reasoning Benchmark - Finance","nStates":38,"nModels":38,"difficulty":159.3,"saturation":245.9,"saturationExtrapolated":true,"frontierPct":55.2,"extrapolated":false,"extrapolation":null,"maxTested":166.1,"saturationDate":"2031-10-22","saturationDateLo":"2030-11-02","saturationDateHi":"2033-01-18","saturated":false},{"slug":"public-benefits-bench-neither","family":"public-benefits-bench-neither","name":"Public Benefits Bench v1 - Neither","nStates":13,"nModels":13,"difficulty":159.3,"saturation":197.6,"saturationExtrapolated":true,"frontierPct":62.9,"extrapolated":false,"extrapolation":null,"maxTested":164.3,"saturationDate":"2028-06-28","saturationDateLo":"2028-01-18","saturationDateHi":"2029-03-01","saturated":false},{"slug":"prinzbench","family":"prinzbench","name":"PrinzBench","nStates":28,"nModels":26,"difficulty":159.4,"saturation":180.5,"saturationExtrapolated":true,"frontierPct":76.8,"extrapolated":false,"extrapolation":null,"maxTested":165.7,"saturationDate":"2027-03-06","saturationDateLo":"2026-09-02","saturationDateHi":"2027-12-09","saturated":false},{"slug":"txbench-pp-v2","family":"txbench-pp-v2","name":"TxBench Preclinical Pharmacology V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":159.5,"saturation":227.2,"saturationExtrapolated":true,"frontierPct":59.3,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2029-08-02","saturationDateLo":"2028-05-03","saturationDateHi":"2034-02-19","saturated":false},{"slug":"biosecbench-surveillance-benchmarks-bio","family":"biosecbench-surveillance-benchmarks-bio","name":"BioSecBench-Surveillance","nStates":17,"nModels":17,"difficulty":159.6,"saturation":180.0,"saturationExtrapolated":true,"frontierPct":70.7,"extrapolated":false,"extrapolation":null,"maxTested":161.8,"saturationDate":"2027-06-08","saturationDateLo":"2027-02-18","saturationDateHi":"2027-10-30","saturated":false},{"slug":"gso","family":"gso","name":"GSO Opt@1","nStates":31,"nModels":29,"difficulty":159.6,"saturation":171.9,"saturationExtrapolated":true,"frontierPct":79.6,"extrapolated":false,"extrapolation":null,"maxTested":166.3,"saturationDate":"2027-01-03","saturationDateLo":"2026-09-02","saturationDateHi":"2027-02-26","saturated":false},{"slug":"ioi-vals-v2","family":"ioi-vals-v2","name":"IOI (Vals v2)","nStates":44,"nModels":43,"difficulty":159.8,"saturation":170.5,"saturationExtrapolated":true,"frontierPct":88.2,"extrapolated":false,"extrapolation":null,"maxTested":171.0,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2026-11-11","saturated":true},{"slug":"turing-vlmbench","family":"turing-vlmbench","name":"Turing VLMBench","nStates":20,"nModels":17,"difficulty":159.9,"saturation":207.5,"saturationExtrapolated":true,"frontierPct":59.5,"extrapolated":true,"extrapolation":"above","maxTested":147.1,"saturationDate":"2029-03-01","saturationDateLo":"2028-03-25","saturationDateHi":"2030-08-25","saturated":false},{"slug":"chessbench","family":"chessbench","name":"ChessBench","nStates":28,"nModels":14,"difficulty":160.0,"saturation":197.6,"saturationExtrapolated":true,"frontierPct":65.9,"extrapolated":false,"extrapolation":null,"maxTested":172.5,"saturationDate":"2028-04-18","saturationDateLo":"2027-10-04","saturationDateHi":"2029-03-29","saturated":false},{"slug":"frontier-financial-judgement","family":"frontier-financial-judgement","name":"Frontier Financial Judgement","nStates":13,"nModels":13,"difficulty":160.1,"saturation":184.8,"saturationExtrapolated":true,"frontierPct":66.0,"extrapolated":false,"extrapolation":null,"maxTested":161.2,"saturationDate":"2027-10-05","saturationDateLo":"2027-05-18","saturationDateHi":"2028-05-12","saturated":false},{"slug":"jobbench-main","family":"jobbench-main","name":"JobBench Main","nStates":22,"nModels":22,"difficulty":160.3,"saturation":181.5,"saturationExtrapolated":true,"frontierPct":68.1,"extrapolated":false,"extrapolation":null,"maxTested":164.9,"saturationDate":"2027-07-20","saturationDateLo":"2027-03-21","saturationDateHi":"2028-04-01","saturated":false},{"slug":"toloka-arena-v3","family":"toloka-arena-v3","name":"Toloka Arena (v3)","nStates":24,"nModels":24,"difficulty":160.4,"saturation":177.1,"saturationExtrapolated":true,"frontierPct":74.2,"extrapolated":false,"extrapolation":null,"maxTested":166.3,"saturationDate":"2027-03-18","saturationDateLo":"2026-12-03","saturationDateHi":"2027-10-14","saturated":false},{"slug":"automationbench-aa-1-0-6","family":"automationbench-aa-1-0-6","name":"AutomationBench-AA 1.0.6","nStates":221,"nModels":146,"difficulty":160.5,"saturation":174.8,"saturationExtrapolated":true,"frontierPct":74.6,"extrapolated":false,"extrapolation":null,"maxTested":168.2,"saturationDate":"2027-03-02","saturationDateLo":"2027-01-02","saturationDateHi":"2027-04-14","saturated":false},{"slug":"babyvision","family":"babyvision","name":"BabyVision","nStates":52,"nModels":44,"difficulty":160.6,"saturation":195.4,"saturationExtrapolated":true,"frontierPct":62.3,"extrapolated":false,"extrapolation":null,"maxTested":168.7,"saturationDate":"2028-05-14","saturationDateLo":"2027-10-06","saturationDateHi":"2029-03-11","saturated":false},{"slug":"code2video-aa","family":"code2video-aa","name":"Code2Video-AA Elo","nStates":27,"nModels":27,"difficulty":160.6,"saturation":172.5,"saturationExtrapolated":true,"frontierPct":77.6,"extrapolated":false,"extrapolation":null,"maxTested":168.2},{"slug":"code-migration-v1-1","family":"code-migration-v1-1","name":"Code Migration v1.1","nStates":27,"nModels":27,"difficulty":160.7,"saturation":172.3,"saturationExtrapolated":true,"frontierPct":77.9,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-01-04","saturationDateLo":"2026-11-08","saturationDateHi":"2027-03-03","saturated":false},{"slug":"enterprisebench-corecraft-surge-ai-2","family":"enterprisebench-corecraft-surge-ai-2","name":"EnterpriseBench: CoreCraft","nStates":43,"nModels":43,"difficulty":160.7,"saturation":181.0,"saturationExtrapolated":true,"frontierPct":67.1,"extrapolated":false,"extrapolation":null,"maxTested":167.9,"saturationDate":"2027-07-24","saturationDateLo":"2027-04-18","saturationDateHi":"2027-11-21","saturated":false},{"slug":"frontiercode-extended-score-cognition-2","family":"frontiercode-extended-score-cognition-2","name":"FrontierCode 1.0 Extended - Score","nStates":41,"nModels":15,"difficulty":160.7,"saturation":182.8,"saturationExtrapolated":true,"frontierPct":65.6,"extrapolated":false,"extrapolation":null,"maxTested":164.2,"saturationDate":"2027-09-18","saturationDateLo":"2027-07-03","saturationDateHi":"2027-11-23","saturated":false},{"slug":"frontiermath-tier-4-v2-private","family":"frontiermath-tier-4-v2-private","name":"FrontierMath Tier 4 (v2) - Private","nStates":84,"nModels":69,"difficulty":160.7,"saturation":170.9,"saturationExtrapolated":true,"frontierPct":86.7,"extrapolated":false,"extrapolation":null,"maxTested":170.9,"saturationDate":"2026-09-02","saturationDateLo":"2026-09-02","saturationDateHi":"2026-11-18","saturated":true},{"slug":"drugdiscoverybench","family":"drugdiscoverybench","name":"DrugDiscoveryBench","nStates":18,"nModels":18,"difficulty":160.9,"saturation":182.9,"saturationExtrapolated":true,"frontierPct":66.5,"extrapolated":false,"extrapolation":null,"maxTested":167.6,"saturationDate":"2027-08-22","saturationDateLo":"2027-05-05","saturationDateHi":"2028-01-27","saturated":false},{"slug":"swe-bench-pro-private-dataset-scale-ai-2","family":"swe-bench-pro-private-dataset-scale-ai-2","name":"SWE-Bench Pro (Private Dataset)","nStates":12,"nModels":12,"difficulty":160.9,"saturation":180.9,"saturationExtrapolated":true,"frontierPct":68.9,"extrapolated":true,"extrapolation":"above","maxTested":158.5,"saturationDate":"2027-05-27","saturationDateLo":"2026-12-19","saturationDateHi":"2028-04-01","saturated":false},{"slug":"professional-reasoning-benchmark-legal-scale-ai-2","family":"professional-reasoning-benchmark-legal-scale-ai-2","name":"Professional Reasoning Benchmark - Legal","nStates":43,"nModels":43,"difficulty":161.0,"saturation":258.6,"saturationExtrapolated":true,"frontierPct":53.8,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2032-07-15","saturationDateLo":"2031-05-10","saturationDateHi":"2034-03-11","saturated":false},{"slug":"simpleqa-verified-epoch-anti-abstention","family":"simpleqa-verified-epoch-anti-abstention","name":"SimpleQA Verified (Epoch anti-abstention)","nStates":78,"nModels":72,"difficulty":161.0,"saturation":200.4,"saturationExtrapolated":true,"frontierPct":64.0,"extrapolated":false,"extrapolation":null,"maxTested":173.4,"saturationDate":"2028-06-28","saturationDateLo":"2028-02-10","saturationDateHi":"2028-12-29","saturated":false},{"slug":"astabench-2026-e2e-hard","family":"astabench-2026-e2e-hard","name":"E2E-Bench-Hard (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":161.1,"saturation":191.3,"saturationExtrapolated":true,"frontierPct":62.7,"extrapolated":true,"extrapolation":"above","maxTested":160.8,"saturationDate":"2028-01-10","saturationDateLo":"2027-05-29","saturationDateHi":"2029-04-04","saturated":false},{"slug":"autoformbench","family":"autoformbench","name":"AutoformBench","nStates":13,"nModels":9,"difficulty":161.1,"saturation":187.4,"saturationExtrapolated":true,"frontierPct":70.1,"extrapolated":false,"extrapolation":null,"maxTested":171.7,"saturationDate":"2027-07-21","saturationDateLo":"2027-03-10","saturationDateHi":"2028-06-22","saturated":false},{"slug":"hle-verified-1811","family":"hle-verified-1811","name":"Humanity's Last Exam - full verified 1,811-item set","nStates":12,"nModels":12,"difficulty":161.1,"saturation":229.9,"saturationExtrapolated":true,"frontierPct":56.5,"extrapolated":false,"extrapolation":null,"maxTested":164.6,"saturationDate":"2030-01-26","saturationDateLo":"2028-08-25","saturationDateHi":"2033-08-16","saturated":false},{"slug":"osworld-2-500-step-partial-score","family":"osworld-2","name":"OSWorld 2.0","nStates":41,"nModels":13,"difficulty":161.1,"saturation":174.2,"saturationExtrapolated":true,"frontierPct":77.9,"extrapolated":false,"extrapolation":null,"maxTested":165.4,"saturationDate":"2027-01-15","saturationDateLo":"2026-11-21","saturationDateHi":"2027-03-12","saturated":false},{"slug":"toloka-arena-v1","family":"toloka-arena-v1","name":"Toloka Arena (v1)","nStates":37,"nModels":33,"difficulty":161.1,"saturation":178.8,"saturationExtrapolated":true,"frontierPct":70.5,"extrapolated":false,"extrapolation":null,"maxTested":166.3,"saturationDate":"2027-05-23","saturationDateLo":"2027-02-23","saturationDateHi":"2027-08-26","saturated":false},{"slug":"aa-3d-v1","family":"aa-3d-v1","name":"AA-3D v1.0 Elo","nStates":23,"nModels":23,"difficulty":161.5,"saturation":174.0,"saturationExtrapolated":true,"frontierPct":75.8,"extrapolated":false,"extrapolation":null,"maxTested":168.6},{"slug":"gba-eval-overall-gbaeval","family":"gba-eval-overall-gbaeval","name":"GBA Eval Overall","nStates":22,"nModels":22,"difficulty":161.7,"saturation":171.3,"saturationExtrapolated":true,"frontierPct":81.9,"extrapolated":false,"extrapolation":null,"maxTested":165.0,"saturationDate":"2026-09-21","saturationDateLo":"2026-09-01","saturationDateHi":"2027-04-21","saturated":true},{"slug":"bizagentbench","family":"bizagentbench","name":"BizAgentBench (strict success)","nStates":8,"nModels":8,"difficulty":161.9,"saturation":171.5,"saturationExtrapolated":true,"frontierPct":82.9,"extrapolated":false,"extrapolation":null,"maxTested":164.6,"saturationDate":"2026-11-09","saturationDateLo":"2026-09-02","saturationDateHi":"2027-03-03","saturated":false},{"slug":"frontiermath-tiers-1-3-v1-private","family":"frontiermath-tiers-1-3-v1-private","name":"FrontierMath Tiers 1-3 (v1) - Private","nStates":8,"nModels":8,"difficulty":162.0,"saturation":198.2,"saturationExtrapolated":true,"frontierPct":59.8,"extrapolated":false,"extrapolation":null,"maxTested":163.3,"saturationDate":"2028-06-21","saturationDateLo":"2027-11-09","saturationDateHi":"2029-09-29","saturated":false},{"slug":"hil-bench-human-in-loop-benchmark-scale-ai-2","family":"hil-bench-human-in-loop-benchmark-scale-ai-2","name":"HiL-Bench (Human-in-Loop Benchmark)","nStates":27,"nModels":27,"difficulty":162.0,"saturation":181.0,"saturationExtrapolated":true,"frontierPct":67.7,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-06-21","saturationDateLo":"2027-02-10","saturationDateHi":"2027-11-30","saturated":false},{"slug":"swe-atlas-test-writing-scale-ai-2","family":"swe-atlas-test-writing-scale-ai-2","name":"SWE Atlas - Test Writing","nStates":23,"nModels":23,"difficulty":162.0,"saturation":188.1,"saturationExtrapolated":true,"frontierPct":60.8,"extrapolated":false,"extrapolation":null,"maxTested":166.5,"saturationDate":"2028-01-02","saturationDateLo":"2027-09-28","saturationDateHi":"2028-05-23","saturated":false},{"slug":"sweeper-bench","family":"sweeper-bench","name":"SWEeper-Bench","nStates":12,"nModels":12,"difficulty":162.0,"saturation":241.7,"saturationExtrapolated":true,"frontierPct":55.7,"extrapolated":false,"extrapolation":null,"maxTested":167.8,"saturationDate":"2030-06-25","saturationDateLo":"2028-07-06","saturationDateHi":"2036-08-16","saturated":false},{"slug":"odysseys2","family":"odysseys2","name":"Odysseys2","nStates":18,"nModels":14,"difficulty":162.1,"saturation":203.3,"saturationExtrapolated":true,"frontierPct":60.8,"extrapolated":false,"extrapolation":null,"maxTested":167.4,"saturationDate":"2028-04-28","saturationDateLo":"2027-05-25","saturationDateHi":"2030-08-29","saturated":false},{"slug":"swe-atlas-refactoring-scale-ai-2","family":"swe-atlas-refactoring-scale-ai-2","name":"SWE Atlas - Refactoring","nStates":18,"nModels":18,"difficulty":162.2,"saturation":186.0,"saturationExtrapolated":true,"frontierPct":61.8,"extrapolated":false,"extrapolation":null,"maxTested":166.9,"saturationDate":"2027-11-20","saturationDateLo":"2027-07-08","saturationDateHi":"2028-04-07","saturated":false},{"slug":"apex-accounting","family":"apex-accounting","name":"APEX-Accounting","nStates":34,"nModels":33,"difficulty":162.3,"saturation":186.4,"saturationExtrapolated":true,"frontierPct":61.7,"extrapolated":false,"extrapolation":null,"maxTested":168.3,"saturationDate":"2027-11-22","saturationDateLo":"2027-08-09","saturationDateHi":"2028-04-07","saturated":false},{"slug":"autoresearchexam","family":"autoresearchexam","name":"AutoResearchExam (hidden-test AUARC)","nStates":11,"nModels":11,"difficulty":162.3,"saturation":193.8,"saturationExtrapolated":true,"frontierPct":60.7,"extrapolated":false,"extrapolation":null,"maxTested":168.5,"saturationDate":"2028-03-11","saturationDateLo":"2027-08-31","saturationDateHi":"2029-03-09","saturated":false},{"slug":"hle-mercor-extended","family":"hle-mercor-extended","name":"HLE Extended (Mercor held-out)","nStates":28,"nModels":28,"difficulty":162.3,"saturation":173.6,"saturationExtrapolated":true,"frontierPct":78.4,"extrapolated":false,"extrapolation":null,"maxTested":166.0,"saturationDate":"2026-12-24","saturationDateLo":"2026-09-02","saturationDateHi":"2027-03-11","saturated":false},{"slug":"apex-swe-mercor-2","family":"apex-swe-mercor-2","name":"APEX-SWE","nStates":71,"nModels":62,"difficulty":162.5,"saturation":190.1,"saturationExtrapolated":true,"frontierPct":59.1,"extrapolated":false,"extrapolation":null,"maxTested":167.6,"saturationDate":"2028-03-14","saturationDateLo":"2027-12-05","saturationDateHi":"2028-06-18","saturated":false},{"slug":"usamo-2025-matharena","family":"usamo-2025-matharena","name":"USAMO 2025","nStates":10,"nModels":10,"difficulty":162.5,"saturation":187.2,"saturationExtrapolated":true,"frontierPct":66.7,"extrapolated":true,"extrapolation":"above","maxTested":149.0,"saturationDate":"2027-08-15","saturationDateLo":"2026-06-09","saturationDateHi":"2029-08-27","saturated":false},{"slug":"frontierswe-dart-style-haskell","family":"frontierswe","name":"FrontierSWE (Dominance)","nStates":15,"nModels":15,"difficulty":162.7,"saturation":168.7,"saturationExtrapolated":false,"frontierPct":99.5,"extrapolated":false,"extrapolation":null,"maxTested":172.6,"saturationDate":"2025-12-11","saturationDateLo":"2025-06-16","saturationDateHi":"2026-04-07","saturated":true},{"slug":"ebr-bench-v4-card-ban","family":"ebr-bench-v4-card-ban","name":"EBR-bench v4 (card ban, single agent)","nStates":11,"nModels":11,"difficulty":162.8,"saturation":198.6,"saturationExtrapolated":true,"frontierPct":65.9,"extrapolated":false,"extrapolation":null,"maxTested":170.7,"saturationDate":"2027-10-13","saturationDateLo":"2027-02-25","saturationDateHi":"2030-09-17","saturated":false},{"slug":"textquests-game-progress","family":"textquests-game-progress","name":"TextQuests Game Progress","nStates":62,"nModels":62,"difficulty":162.9,"saturation":188.0,"saturationExtrapolated":true,"frontierPct":66.4,"extrapolated":false,"extrapolation":null,"maxTested":170.4,"saturationDate":"2027-09-29","saturationDateLo":"2027-07-03","saturationDateHi":"2028-01-23","saturated":false},{"slug":"code-migration","family":"code-migration","name":"Code Migration","nStates":73,"nModels":72,"difficulty":163.0,"saturation":177.6,"saturationExtrapolated":true,"frontierPct":66.0,"extrapolated":false,"extrapolation":null,"maxTested":167.4,"saturationDate":"2027-05-09","saturationDateLo":"2027-03-07","saturationDateHi":"2027-07-07","saturated":false},{"slug":"itbench-aa-artificial-analysis","family":"itbench-aa-artificial-analysis","name":"ITBench-AA","nStates":44,"nModels":42,"difficulty":163.0,"saturation":193.7,"saturationExtrapolated":true,"frontierPct":57.5,"extrapolated":false,"extrapolation":null,"maxTested":168.0,"saturationDate":"2028-05-16","saturationDateLo":"2028-01-09","saturationDateHi":"2028-10-23","saturated":false},{"slug":"opus-magnum-campaign-opus-magnum-bench-2","family":"opus-magnum-campaign-opus-magnum-bench-2","name":"Opus Magnum Campaign","nStates":23,"nModels":20,"difficulty":163.2,"saturation":174.5,"saturationExtrapolated":true,"frontierPct":75.0,"extrapolated":false,"extrapolation":null,"maxTested":164.0,"saturationDate":"2027-01-12","saturationDateLo":"2026-09-02","saturationDateHi":"2027-04-27","saturated":false},{"slug":"pointerbench","family":"pointerbench","name":"Pointerbench","nStates":29,"nModels":29,"difficulty":163.3,"saturation":179.9,"saturationExtrapolated":true,"frontierPct":66.2,"extrapolated":false,"extrapolation":null,"maxTested":163.8,"saturationDate":"2027-05-08","saturationDateLo":"2026-12-09","saturationDateHi":"2028-03-06","saturated":false},{"slug":"skillsbench-without-skills","family":"skillsbench-without-skills","name":"SkillsBench without Skills","nStates":51,"nModels":45,"difficulty":163.4,"saturation":188.2,"saturationExtrapolated":true,"frontierPct":60.2,"extrapolated":false,"extrapolation":null,"maxTested":163.9,"saturationDate":"2028-01-03","saturationDateLo":"2027-09-21","saturationDateHi":"2028-04-28","saturated":false},{"slug":"hle","family":"hle","name":"Humanity's Last Exam","nStates":213,"nModels":130,"difficulty":163.5,"saturation":185.1,"saturationExtrapolated":true,"frontierPct":64.0,"extrapolated":false,"extrapolation":null,"maxTested":168.2,"saturationDate":"2027-10-09","saturationDateLo":"2027-08-02","saturationDateHi":"2027-12-02","saturated":false},{"slug":"exploitbench-leaderboard-mean-capability-score-out-of-16-exploitbench","family":"exploitbench-leaderboard-mean-capability-score-out-of-16-exploitbench","name":"ExploitBench Leaderboard (Mean capability score, out of 16)","nStates":13,"nModels":13,"difficulty":163.6,"saturation":180.9,"saturationExtrapolated":true,"frontierPct":64.3,"extrapolated":false,"extrapolation":null,"maxTested":165.4,"saturationDate":"2027-06-21","saturationDateLo":"2027-03-04","saturationDateHi":"2027-11-15","saturated":false},{"slug":"medcode","family":"medcode","name":"MedCode","nStates":104,"nModels":94,"difficulty":163.6,"saturation":241.9,"saturationExtrapolated":true,"frontierPct":53.3,"extrapolated":false,"extrapolation":null,"maxTested":168.9,"saturationDate":"2031-06-21","saturationDateLo":"2030-09-12","saturationDateHi":"2032-05-29","saturated":false},{"slug":"p2npcbench","family":"p2npcbench","name":"P2NPCBench (NP-complete, pass@1)","nStates":11,"nModels":11,"difficulty":163.6,"saturation":231.7,"saturationExtrapolated":true,"frontierPct":54.1,"extrapolated":true,"extrapolation":"above","maxTested":161.6,"saturationDate":"2030-07-29","saturationDateLo":"2029-03-07","saturationDateHi":"2032-11-22","saturated":false},{"slug":"worldvqa-accuracy","family":"worldvqa-accuracy","name":"WorldVQA Accuracy","nStates":21,"nModels":17,"difficulty":163.6,"saturation":208.4,"saturationExtrapolated":true,"frontierPct":55.7,"extrapolated":true,"extrapolation":"above","maxTested":161.6,"saturationDate":"2029-02-25","saturationDateLo":"2028-04-07","saturationDateHi":"2030-09-20","saturated":false},{"slug":"swe-atlas-codebase-qna-scale-ai-2","family":"swe-atlas-codebase-qna-scale-ai-2","name":"SWE Atlas - Codebase QnA","nStates":68,"nModels":53,"difficulty":163.7,"saturation":189.6,"saturationExtrapolated":true,"frontierPct":57.5,"extrapolated":false,"extrapolation":null,"maxTested":167.8,"saturationDate":"2028-02-17","saturationDateLo":"2027-10-27","saturationDateHi":"2028-07-19","saturated":false},{"slug":"cad-arena","family":"cad-arena","name":"CAD Arena","nStates":12,"nModels":12,"difficulty":163.9,"saturation":179.5,"saturationExtrapolated":true,"frontierPct":66.1,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-04-27","saturationDateLo":"2027-01-03","saturationDateHi":"2027-10-12","saturated":false},{"slug":"legal_research","family":"legal_research","name":"Legal Research Bench","nStates":78,"nModels":77,"difficulty":164.1,"saturation":184.1,"saturationExtrapolated":true,"frontierPct":58.3,"extrapolated":false,"extrapolation":null,"maxTested":167.5,"saturationDate":"2027-10-23","saturationDateLo":"2027-08-08","saturationDateHi":"2028-01-05","saturated":false},{"slug":"furniture-assembly","family":"furniture-assembly","name":"Furniture Assembly Benchmark","nStates":35,"nModels":35,"difficulty":164.3,"saturation":178.8,"saturationExtrapolated":true,"frontierPct":70.6,"extrapolated":false,"extrapolation":null,"maxTested":171.8,"saturationDate":"2027-03-19","saturationDateLo":"2027-01-13","saturationDateHi":"2027-05-29","saturated":false},{"slug":"mirrorcode-ml-private-2l","family":"mirrorcode-ml-private-2l","name":"MirrorCode (ML, +Private, 2L)","nStates":15,"nModels":15,"difficulty":164.3,"saturation":176.7,"saturationExtrapolated":true,"frontierPct":63.4,"extrapolated":false,"extrapolation":null,"maxTested":167.3,"saturationDate":"2027-04-08","saturationDateLo":"2027-01-10","saturationDateHi":"2027-09-03","saturated":false},{"slug":"long-horizon-terminal-bench-1-0","family":"long-horizon-terminal-bench-1-0","name":"Long-Horizon Terminal-Bench 1.0 (mean reward)","nStates":21,"nModels":21,"difficulty":164.4,"saturation":194.9,"saturationExtrapolated":true,"frontierPct":57.7,"extrapolated":true,"extrapolation":"above","maxTested":163.8,"saturationDate":"2028-04-02","saturationDateLo":"2027-09-30","saturationDateHi":"2029-03-12","saturated":false},{"slug":"clockbench-accuracy","family":"clockbench-accuracy","name":"ClockBench Accuracy","nStates":38,"nModels":38,"difficulty":164.5,"saturation":188.0,"saturationExtrapolated":true,"frontierPct":62.6,"extrapolated":false,"extrapolation":null,"maxTested":171.6,"saturationDate":"2027-10-21","saturationDateLo":"2027-06-29","saturationDateHi":"2028-03-23","saturated":false},{"slug":"visualtoolbench-vtb-scale-ai-2","family":"visualtoolbench-vtb-scale-ai-2","name":"VisualToolBench (VTB)","nStates":21,"nModels":17,"difficulty":164.5,"saturation":183.4,"saturationExtrapolated":true,"frontierPct":60.9,"extrapolated":true,"extrapolation":"above","maxTested":158.5,"saturationDate":"2027-08-20","saturationDateLo":"2027-04-24","saturationDateHi":"2028-01-31","saturated":false},{"slug":"aa-analyst-agent-pass-caret-5","family":"aa-analyst-agent-pass-caret-5","name":"AA-AnalystAgent (pass^5)","nStates":50,"nModels":48,"difficulty":164.6,"saturation":185.4,"saturationExtrapolated":true,"frontierPct":59.5,"extrapolated":false,"extrapolation":null,"maxTested":169.0,"saturationDate":"2027-10-21","saturationDateLo":"2027-08-01","saturationDateHi":"2028-01-14","saturated":false},{"slug":"biosecbench-surveillance-v2","family":"biosecbench-surveillance-v2","name":"BioSecBench-Surveillance V1 Clean Reruns","nStates":23,"nModels":23,"difficulty":164.6,"saturation":200.8,"saturationExtrapolated":true,"frontierPct":56.1,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2028-08-16","saturationDateLo":"2027-12-29","saturationDateHi":"2029-08-14","saturated":false},{"slug":"physicianbench","family":"physicianbench","name":"PhysicianBench","nStates":13,"nModels":13,"difficulty":164.7,"saturation":182.1,"saturationExtrapolated":true,"frontierPct":58.8,"extrapolated":true,"extrapolation":"above","maxTested":161.6,"saturationDate":"2027-06-18","saturationDateLo":"2027-01-23","saturationDateHi":"2028-04-12","saturated":false},{"slug":"chess-puzzles-v2","family":"chess-puzzles-v2","name":"Chess Puzzles (task version 2.0)","nStates":149,"nModels":81,"difficulty":164.8,"saturation":192.5,"saturationExtrapolated":true,"frontierPct":60.9,"extrapolated":false,"extrapolation":null,"maxTested":172.3,"saturationDate":"2028-02-04","saturationDateLo":"2027-11-07","saturationDateHi":"2028-05-04","saturated":false},{"slug":"riemannbench","family":"riemannbench","name":"RiemannBench","nStates":39,"nModels":39,"difficulty":164.8,"saturation":178.8,"saturationExtrapolated":true,"frontierPct":73.9,"extrapolated":false,"extrapolation":null,"maxTested":173.6,"saturationDate":"2027-02-11","saturationDateLo":"2026-12-03","saturationDateHi":"2027-04-20","saturated":false},{"slug":"enterpriseops-gym-aa-artificial-analysis","family":"enterpriseops-gym-aa-artificial-analysis","name":"EnterpriseOps-Gym-AA","nStates":47,"nModels":45,"difficulty":165.0,"saturation":216.9,"saturationExtrapolated":true,"frontierPct":52.8,"extrapolated":false,"extrapolation":null,"maxTested":165.1,"saturationDate":"2029-10-17","saturationDateLo":"2029-03-30","saturationDateHi":"2030-07-17","saturated":false},{"slug":"kernelbench-cuda-normalized-rtx-pro-6000","family":"kernelbench-cuda-normalized-rtx-pro-6000","name":"KernelBench CUDA Normalized (RTX PRO 6000)","nStates":20,"nModels":20,"difficulty":165.1,"saturation":181.2,"saturationExtrapolated":true,"frontierPct":61.5,"extrapolated":false,"extrapolation":null,"maxTested":168.7,"saturationDate":"2027-05-05","saturationDateLo":"2027-01-28","saturationDateHi":"2027-12-20","saturated":false},{"slug":"swe-bench-multimodal-vals-v1","family":"swe-bench-multimodal","name":"SWE-bench Multimodal","nStates":19,"nModels":19,"difficulty":165.2,"saturation":200.1,"saturationExtrapolated":true,"frontierPct":53.9,"extrapolated":false,"extrapolation":null,"maxTested":168.3,"saturationDate":"2028-08-18","saturationDateLo":"2028-01-16","saturationDateHi":"2029-08-10","saturated":false},{"slug":"terminal-bench-4-0","family":"terminal-bench-4-0","name":"Terminal-Bench 4.0","nStates":115,"nModels":63,"difficulty":165.2,"saturation":173.4,"saturationExtrapolated":true,"frontierPct":63.4,"extrapolated":false,"extrapolation":null,"maxTested":168.1,"saturationDate":"2027-02-05","saturationDateLo":"2026-12-06","saturationDateHi":"2027-03-14","saturated":false},{"slug":"complexconstraints-surge-ai-2","family":"complexconstraints-surge-ai-2","name":"ComplexConstraints","nStates":37,"nModels":37,"difficulty":165.3,"saturation":190.7,"saturationExtrapolated":true,"frontierPct":56.8,"extrapolated":false,"extrapolation":null,"maxTested":169.4,"saturationDate":"2028-01-26","saturationDateLo":"2027-10-30","saturationDateHi":"2028-06-01","saturated":false},{"slug":"browsecomp-mercor-extended","family":"browsecomp-mercor-extended","name":"BrowseComp Extended (Mercor held-out)","nStates":35,"nModels":35,"difficulty":165.4,"saturation":177.2,"saturationExtrapolated":true,"frontierPct":60.5,"extrapolated":false,"extrapolation":null,"maxTested":168.4,"saturationDate":"2027-04-20","saturationDateLo":"2027-02-07","saturationDateHi":"2027-07-07","saturated":false},{"slug":"cursorbench-4-0","family":"cursorbench-4-0","name":"CursorBench 4.0","nStates":84,"nModels":19,"difficulty":165.4,"saturation":186.2,"saturationExtrapolated":true,"frontierPct":56.2,"extrapolated":false,"extrapolation":null,"maxTested":168.4,"saturationDate":"2027-11-24","saturationDateLo":"2027-09-24","saturationDateHi":"2028-01-20","saturated":false},{"slug":"reactbench-v1","family":"reactbench-v1","name":"ReactBench v1","nStates":58,"nModels":16,"difficulty":165.4,"saturation":185.5,"saturationExtrapolated":true,"frontierPct":57.4,"extrapolated":true,"extrapolation":"above","maxTested":165.2,"saturationDate":"2027-10-31","saturationDateLo":"2027-07-27","saturationDateHi":"2028-02-12","saturated":false},{"slug":"samaya-frontierfinance-v3-1","family":"samaya-frontierfinance-v3-1","name":"FrontierFinance v3.1","nStates":13,"nModels":13,"difficulty":165.5,"saturation":234.6,"saturationExtrapolated":true,"frontierPct":52.2,"extrapolated":false,"extrapolation":null,"maxTested":165.7,"saturationDate":"2030-05-26","saturationDateLo":"2028-10-22","saturationDateHi":"2033-06-25","saturated":false},{"slug":"biosecbench-function-v2","family":"biosecbench-function-v2","name":"BioSecBench-Function V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":165.6,"saturation":185.5,"saturationExtrapolated":true,"frontierPct":57.7,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-09-27","saturationDateLo":"2027-05-20","saturationDateHi":"2028-03-01","saturated":false},{"slug":"westworld-finance-diligence","family":"westworld-finance-diligence","name":"Westworld Finance Diligence Bench","nStates":9,"nModels":9,"difficulty":165.6,"saturation":196.8,"saturationExtrapolated":true,"frontierPct":56.9,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-11-21","saturationDateLo":"2027-04-06","saturationDateHi":"2030-02-03","saturated":false},{"slug":"frontierswe-v2","family":"frontierswe-v2","name":"FrontierSWE V2","nStates":19,"nModels":19,"difficulty":165.7,"saturation":174.8,"saturationExtrapolated":true,"frontierPct":59.3,"extrapolated":false,"extrapolation":null,"maxTested":167.9,"saturationDate":"2027-03-07","saturationDateLo":"2027-01-03","saturationDateHi":"2027-05-04","saturated":false},{"slug":"revelo-code-index","family":"revelo-code-index","name":"Revelo Code Index","nStates":19,"nModels":19,"difficulty":165.8,"saturation":183.4,"saturationExtrapolated":true,"frontierPct":54.3,"extrapolated":false,"extrapolation":null,"maxTested":166.5,"saturationDate":"2027-09-17","saturationDateLo":"2027-06-22","saturationDateHi":"2028-01-10","saturated":false},{"slug":"terminal-bench-4-0-artificial-analysis","family":"terminal-bench-4-0-artificial-analysis","name":"Terminal-Bench 4.0 (Artificial Analysis)","nStates":227,"nModels":146,"difficulty":165.8,"saturation":174.1,"saturationExtrapolated":true,"frontierPct":61.8,"extrapolated":false,"extrapolation":null,"maxTested":168.4,"saturationDate":"2027-02-14","saturationDateLo":"2026-12-22","saturationDateHi":"2027-03-22","saturated":false},{"slug":"shroombench","family":"shroombench","name":"Shroombench (mushroom identification, top-1)","nStates":21,"nModels":21,"difficulty":165.9,"saturation":211.2,"saturationExtrapolated":true,"frontierPct":53.4,"extrapolated":false,"extrapolation":null,"maxTested":166.1,"saturationDate":"2028-12-29","saturationDateLo":"2027-12-05","saturationDateHi":"2031-09-12","saturated":false},{"slug":"active-probing-benchmark","family":"active-probing-benchmark","name":"Active Probing Benchmark (APB)","nStates":8,"nModels":8,"difficulty":166.0,"saturation":202.4,"saturationExtrapolated":true,"frontierPct":53.6,"extrapolated":true,"extrapolation":"above","maxTested":149.3,"saturationDate":"2028-09-17","saturationDateLo":"2027-02-11","saturationDateHi":"2031-03-16","saturated":false},{"slug":"otelbench","family":"otelbench","name":"OTelBench","nStates":14,"nModels":14,"difficulty":166.0,"saturation":187.3,"saturationExtrapolated":true,"frontierPct":54.4,"extrapolated":true,"extrapolation":"above","maxTested":152.7,"saturationDate":"2027-08-22","saturationDateLo":"2027-01-27","saturationDateHi":"2028-11-21","saturated":false},{"slug":"txbench-ab-v1","family":"txbench-ab-v1","name":"TxBench Antibody Discovery V0","nStates":14,"nModels":14,"difficulty":166.1,"saturation":198.9,"saturationExtrapolated":true,"frontierPct":54.9,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2028-01-29","saturationDateLo":"2027-05-19","saturationDateHi":"2030-09-18","saturated":false},{"slug":"matflowbench","family":"matflowbench","name":"MatFlowBench Pass@1","nStates":9,"nModels":9,"difficulty":166.2,"saturation":176.0,"saturationExtrapolated":true,"frontierPct":56.4,"extrapolated":false,"extrapolation":null,"maxTested":167.3,"saturationDate":"2027-03-16","saturationDateLo":"2027-01-01","saturationDateHi":"2027-06-16","saturated":false},{"slug":"qodo-code-review-benchmark-125","family":"qodo-code-review-benchmark-125","name":"Qodo AI Code Review Benchmark (125 PRs)","nStates":35,"nModels":11,"difficulty":166.2,"saturation":218.5,"saturationExtrapolated":true,"frontierPct":51.9,"extrapolated":false,"extrapolation":null,"maxTested":168.1,"saturationDate":"2029-11-09","saturationDateLo":"2029-01-25","saturationDateHi":"2031-01-08","saturated":false},{"slug":"time-horizon-index-ksp","family":"time-horizon-index-ksp","name":"Time Horizon Index: KSP","nStates":13,"nModels":13,"difficulty":166.3,"saturation":172.3,"saturationExtrapolated":true,"frontierPct":68.7,"extrapolated":false,"extrapolation":null,"maxTested":169.2,"saturationDate":"2026-12-02","saturationDateLo":"2026-09-02","saturationDateHi":"2027-01-27","saturated":false},{"slug":"terminal-bench-science-0-1","family":"terminal-bench-science-0-1","name":"Terminal-Bench-Science 0.1","nStates":120,"nModels":59,"difficulty":166.4,"saturation":171.7,"saturationExtrapolated":true,"frontierPct":60.2,"extrapolated":false,"extrapolation":null,"maxTested":168.3,"saturationDate":"2026-12-23","saturationDateLo":"2026-09-02","saturationDateHi":"2027-01-24","saturated":false},{"slug":"global-piqa-parallel-loglikelihood-accuracy","family":"global-piqa-parallel-loglikelihood-accuracy","name":"Global PIQA - Parallel (Log-Likelihood Accuracy)","nStates":20,"nModels":20,"difficulty":166.5,"saturation":201.7,"saturationExtrapolated":true,"frontierPct":64.8,"extrapolated":true,"extrapolation":"above","maxTested":122.3,"saturationDate":"2028-09-27","saturationDateLo":"2027-03-01","saturationDateHi":"2031-01-08","saturated":false},{"slug":"taxcalcbench-ty25-web-search","family":"taxcalcbench-ty25-web-search","name":"TaxCalcBench TY25 (web search)","nStates":24,"nModels":20,"difficulty":166.5,"saturation":180.6,"saturationExtrapolated":true,"frontierPct":65.8,"extrapolated":false,"extrapolation":null,"maxTested":171.2,"saturationDate":"2027-04-08","saturationDateLo":"2027-01-28","saturationDateHi":"2027-07-14","saturated":false},{"slug":"big-finance-bench-private-holdout","family":"big-finance-bench-private-holdout","name":"BigFinanceBench private holdout (final-answer accuracy)","nStates":14,"nModels":14,"difficulty":166.6,"saturation":220.4,"saturationExtrapolated":true,"frontierPct":52.1,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2029-06-30","saturationDateLo":"2028-04-03","saturationDateHi":"2032-09-03","saturated":false},{"slug":"enigmaeval-scale-ai-2","family":"enigmaeval-scale-ai-2","name":"EnigmaEval","nStates":50,"nModels":48,"difficulty":166.6,"saturation":181.4,"saturationExtrapolated":true,"frontierPct":66.0,"extrapolated":false,"extrapolation":null,"maxTested":170.8,"saturationDate":"2027-04-24","saturationDateLo":"2027-02-26","saturationDateHi":"2027-06-14","saturated":false},{"slug":"chartography-surge-ai-2","family":"chartography-surge-ai-2","name":"Chartography","nStates":45,"nModels":40,"difficulty":166.7,"saturation":183.7,"saturationExtrapolated":true,"frontierPct":58.5,"extrapolated":false,"extrapolation":null,"maxTested":170.8,"saturationDate":"2027-07-31","saturationDateLo":"2027-05-03","saturationDateHi":"2027-11-21","saturated":false},{"slug":"biomysterybench-human-difficult","family":"biomysterybench-human-difficult","name":"BioMysteryBench - Human Difficult","nStates":9,"nModels":9,"difficulty":166.8,"saturation":237.7,"saturationExtrapolated":true,"frontierPct":51.5,"extrapolated":true,"extrapolation":"above","maxTested":165.4,"saturationDate":"2030-04-30","saturationDateLo":"2028-04-22","saturationDateHi":"2035-03-12","saturated":false},{"slug":"rsi-exam-0-1","family":"rsi-exam-0-1","name":"RSI-Exam 0.1 (Full 88)","nStates":17,"nModels":17,"difficulty":166.8,"saturation":191.0,"saturationExtrapolated":true,"frontierPct":50.8,"extrapolated":false,"extrapolation":null,"maxTested":168.0,"saturationDate":"2028-03-08","saturationDateLo":"2027-11-20","saturationDateHi":"2028-07-22","saturated":false},{"slug":"blueprint-bench-2","family":"blueprint-bench-2","name":"Blueprint-Bench 2","nStates":39,"nModels":34,"difficulty":166.9,"saturation":183.9,"saturationExtrapolated":true,"frontierPct":53.6,"extrapolated":false,"extrapolation":null,"maxTested":167.2,"saturationDate":"2027-09-15","saturationDateLo":"2027-06-11","saturationDateHi":"2028-01-30","saturated":false},{"slug":"ramp-accounting-bench","family":"ramp-accounting-bench","name":"Ramp Accounting Bench","nStates":16,"nModels":16,"difficulty":166.9,"saturation":209.8,"saturationExtrapolated":true,"frontierPct":52.1,"extrapolated":false,"extrapolation":null,"maxTested":168.1,"saturationDate":"2029-03-14","saturationDateLo":"2028-06-06","saturationDateHi":"2030-06-13","saturated":false},{"slug":"arxivlean-06-2026-matharena","family":"arxivlean-06-2026-matharena","name":"ArXivLean 06/2026","nStates":8,"nModels":8,"difficulty":167.0,"saturation":177.0,"saturationExtrapolated":true,"frontierPct":58.1,"extrapolated":false,"extrapolation":null,"maxTested":169.4,"saturationDate":"2027-03-23","saturationDateLo":"2027-01-09","saturationDateHi":"2027-06-02","saturated":false},{"slug":"sage","family":"sage","name":"SAGE","nStates":89,"nModels":80,"difficulty":167.0,"saturation":261.1,"saturationExtrapolated":true,"frontierPct":52.5,"extrapolated":false,"extrapolation":null,"maxTested":173.5,"saturationDate":"2032-07-14","saturationDateLo":"2031-04-27","saturationDateHi":"2033-10-17","saturated":false},{"slug":"swe-marathon-v1-1","family":"swe-marathon-v1-1","name":"SWE-Marathon v1.1","nStates":52,"nModels":26,"difficulty":167.0,"saturation":184.3,"saturationExtrapolated":true,"frontierPct":54.1,"extrapolated":false,"extrapolation":null,"maxTested":169.2,"saturationDate":"2027-09-13","saturationDateLo":"2027-05-22","saturationDateHi":"2028-02-09","saturated":false},{"slug":"decision-bench-1-1","family":"decision-bench-1-1","name":"Decision-Bench v1.1","nStates":53,"nModels":16,"difficulty":167.1,"saturation":179.0,"saturationExtrapolated":true,"frontierPct":53.8,"extrapolated":false,"extrapolation":null,"maxTested":168.0,"saturationDate":"2027-06-04","saturationDateLo":"2027-04-05","saturationDateHi":"2027-07-27","saturated":false},{"slug":"chess-puzzles","family":"chess-puzzles","name":"Chess Puzzles","nStates":90,"nModels":73,"difficulty":167.2,"saturation":192.4,"saturationExtrapolated":true,"frontierPct":58.8,"extrapolated":true,"extrapolation":"above","maxTested":166.7,"saturationDate":"2027-12-26","saturationDateLo":"2027-08-28","saturationDateHi":"2028-05-09","saturated":false},{"slug":"atlas-exa-200","family":"atlas-exa-200","name":"ATLAS (Exa, 200-task subset)","nStates":8,"nModels":8,"difficulty":167.4,"saturation":209.8,"saturationExtrapolated":true,"frontierPct":55.0,"extrapolated":false,"extrapolation":null,"maxTested":171.8,"saturationDate":"2028-10-22","saturationDateLo":"2027-10-24","saturationDateHi":"2030-10-25","saturated":false},{"slug":"apex-agents","family":"apex-agents","name":"APEX-Agents","nStates":21,"nModels":19,"difficulty":167.6,"saturation":198.1,"saturationExtrapolated":true,"frontierPct":50.5,"extrapolated":true,"extrapolation":"above","maxTested":164.4,"saturationDate":"2028-07-24","saturationDateLo":"2028-02-10","saturationDateHi":"2029-03-10","saturated":false},{"slug":"mystery-game-puzzles","family":"mystery-game-puzzles","name":"Mystery Game Puzzles","nStates":132,"nModels":74,"difficulty":167.6,"saturation":194.1,"saturationExtrapolated":true,"frontierPct":51.1,"extrapolated":false,"extrapolation":null,"maxTested":168.9,"saturationDate":"2028-05-20","saturationDateLo":"2028-01-26","saturationDateHi":"2028-09-20","saturated":false},{"slug":"terminal-bench-3-reported","family":"terminal-bench-3-reported","name":"Terminal-Bench 3 - release-reported accuracy","nStates":9,"nModels":9,"difficulty":167.6,"saturation":180.3,"saturationExtrapolated":true,"frontierPct":49.8,"extrapolated":true,"extrapolation":"above","maxTested":165.1,"saturationDate":"2027-04-26","saturationDateLo":"2027-01-15","saturationDateHi":"2028-02-27","saturated":false},{"slug":"banking-artificial-analysis","family":"banking-artificial-analysis","name":"\ud835\udf0f\u00b3-Banking","nStates":231,"nModels":175,"difficulty":167.7,"saturation":196.0,"saturationExtrapolated":true,"frontierPct":51.0,"extrapolated":false,"extrapolation":null,"maxTested":168.9,"saturationDate":"2028-06-30","saturationDateLo":"2028-04-02","saturationDateHi":"2028-09-19","saturated":false},{"slug":"frontiermath-tier-4-v1-private","family":"frontiermath-tier-4-v1-private","name":"FrontierMath Tier 4 (v1) - Private","nStates":9,"nModels":9,"difficulty":167.7,"saturation":188.4,"saturationExtrapolated":true,"frontierPct":50.8,"extrapolated":true,"extrapolation":"above","maxTested":163.3,"saturationDate":"2027-11-08","saturationDateLo":"2027-06-25","saturationDateHi":"2028-08-03","saturated":false},{"slug":"healthbench-card-hard","family":"healthbench-card-hard","name":"HealthBench Hard","nStates":25,"nModels":17,"difficulty":167.7,"saturation":215.9,"saturationExtrapolated":true,"frontierPct":50.4,"extrapolated":true,"extrapolation":"above","maxTested":158.7,"saturationDate":"2029-07-28","saturationDateLo":"2028-07-22","saturationDateHi":"2031-05-02","saturated":false},{"slug":"variantbench-v2","family":"variantbench-v2","name":"VariantBench V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":167.7,"saturation":194.2,"saturationExtrapolated":true,"frontierPct":51.5,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2028-04-05","saturationDateLo":"2027-09-23","saturationDateHi":"2028-12-13","saturated":false},{"slug":"astabench-2026-super-expert","family":"astabench-2026-super-expert","name":"SUPER-Expert (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":167.8,"saturation":212.2,"saturationExtrapolated":true,"frontierPct":50.0,"extrapolated":true,"extrapolation":"above","maxTested":160.9,"saturationDate":"2029-05-09","saturationDateLo":"2028-07-06","saturationDateHi":"2030-10-26","saturated":false},{"slug":"genebench-pro","family":"genebench-pro","name":"GeneBench-Pro","nStates":61,"nModels":27,"difficulty":167.8,"saturation":177.0,"saturationExtrapolated":true,"frontierPct":49.3,"extrapolated":true,"extrapolation":"above","maxTested":164.2,"saturationDate":"2027-04-22","saturationDateLo":"2027-02-11","saturationDateHi":"2027-06-22","saturated":false},{"slug":"realm-financial","family":"realm-financial","name":"Realm Financial Reasoning","nStates":17,"nModels":17,"difficulty":167.8,"saturation":240.9,"saturationExtrapolated":true,"frontierPct":50.0,"extrapolated":true,"extrapolation":"above","maxTested":166.7,"saturationDate":"2031-03-17","saturationDateLo":"2030-01-15","saturationDateHi":"2033-02-10","saturated":false},{"slug":"variantbench-benchmarks-bio","family":"variantbench-benchmarks-bio","name":"VariantBench","nStates":18,"nModels":15,"difficulty":167.8,"saturation":200.5,"saturationExtrapolated":true,"frontierPct":50.3,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2028-09-29","saturationDateLo":"2028-03-26","saturationDateHi":"2029-05-25","saturated":false},{"slug":"kernelbench-mega-normalized-h100","family":"kernelbench-mega-normalized-h100","name":"KernelBench Mega Normalized (H100)","nStates":15,"nModels":15,"difficulty":167.9,"saturation":184.7,"saturationExtrapolated":true,"frontierPct":50.1,"extrapolated":true,"extrapolation":"above","maxTested":164.6,"saturationDate":"2027-05-10","saturationDateLo":"2026-09-02","saturationDateHi":"2029-07-05","saturated":false},{"slug":"concept-synth-induction-challenge100-round1","family":"concept-synth-induction-challenge100-round1","name":"Concept Synth INDUCTION Challenge100 Round 1","nStates":28,"nModels":28,"difficulty":168.0,"saturation":179.7,"saturationExtrapolated":true,"frontierPct":58.0,"extrapolated":false,"extrapolation":null,"maxTested":174.6,"saturationDate":"2027-04-07","saturationDateLo":"2027-01-16","saturationDateHi":"2027-08-02","saturated":false},{"slug":"scbench-long-benchmarks-bio","family":"scbench-long-benchmarks-bio","name":"ScBench-Long","nStates":19,"nModels":19,"difficulty":168.0,"saturation":180.7,"saturationExtrapolated":true,"frontierPct":53.6,"extrapolated":true,"extrapolation":"above","maxTested":164.6,"saturationDate":"2027-05-21","saturationDateLo":"2027-01-17","saturationDateHi":"2027-10-26","saturated":false},{"slug":"brokenarxiv-overall-matharena","family":"brokenarxiv-overall-matharena","name":"BrokenArXiv \u2014 Overall","nStates":10,"nModels":10,"difficulty":168.2,"saturation":192.9,"saturationExtrapolated":true,"frontierPct":51.2,"extrapolated":true,"extrapolation":"above","maxTested":164.3,"saturationDate":"2027-12-23","saturationDateLo":"2027-02-04","saturationDateHi":"2029-10-09","saturated":false},{"slug":"hackerrank-bench","family":"hackerrank-bench","name":"HackerRank-Bench","nStates":30,"nModels":15,"difficulty":168.2,"saturation":190.5,"saturationExtrapolated":true,"frontierPct":47.8,"extrapolated":true,"extrapolation":"above","maxTested":167.3,"saturationDate":"2028-03-02","saturationDateLo":"2027-12-08","saturationDateHi":"2028-06-05","saturated":false},{"slug":"reactbench-39","family":"reactbench-39","name":"ReactBench (39 tasks)","nStates":83,"nModels":27,"difficulty":168.2,"saturation":191.0,"saturationExtrapolated":true,"frontierPct":49.8,"extrapolated":false,"extrapolation":null,"maxTested":168.9,"saturationDate":"2028-03-03","saturationDateLo":"2027-12-15","saturationDateHi":"2028-05-29","saturated":false},{"slug":"frontierml","family":"frontierml","name":"FrontierML ($200 wallet)","nStates":18,"nModels":18,"difficulty":168.3,"saturation":179.2,"saturationExtrapolated":true,"frontierPct":44.6,"extrapolated":true,"extrapolation":"above","maxTested":167.8,"saturationDate":"2027-05-23","saturationDateLo":"2027-03-08","saturationDateHi":"2027-09-19","saturated":false},{"slug":"mls-bench-lite","family":"mls-bench-lite","name":"MLS Bench Lite","nStates":8,"nModels":8,"difficulty":168.3,"saturation":218.1,"saturationExtrapolated":true,"frontierPct":49.3,"extrapolated":true,"extrapolation":"above","maxTested":163.6,"saturationDate":"2029-05-10","saturationDateLo":"2028-04-03","saturationDateHi":"2032-03-02","saturated":false},{"slug":"bankertoolbench-1-1","family":"bankertoolbench-1-1","name":"BankerToolBench v1.1","nStates":11,"nModels":11,"difficulty":168.5,"saturation":181.7,"saturationExtrapolated":true,"frontierPct":48.4,"extrapolated":true,"extrapolation":"above","maxTested":164.6,"saturationDate":"2027-07-23","saturationDateLo":"2027-04-18","saturationDateHi":"2027-11-17","saturated":false},{"slug":"metagenomicsbench-v1","family":"metagenomicsbench-v1","name":"MetagenomicsBench V1","nStates":23,"nModels":23,"difficulty":168.5,"saturation":199.4,"saturationExtrapolated":true,"frontierPct":49.7,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2028-06-16","saturationDateLo":"2027-10-23","saturationDateHi":"2029-09-22","saturated":false},{"slug":"real-swe-1-1","family":"real-swe-1-1","name":"Real-SWE 1.1","nStates":13,"nModels":13,"difficulty":168.5,"saturation":189.1,"saturationExtrapolated":true,"frontierPct":47.9,"extrapolated":false,"extrapolation":null,"maxTested":168.5,"saturationDate":"2027-11-18","saturationDateLo":"2027-06-20","saturationDateHi":"2028-09-29","saturated":false},{"slug":"terminal-bench-science-vals-retry","family":"terminal-bench-science-vals-retry","name":"Terminal-Bench Science 0.1 (Vals)","nStates":43,"nModels":41,"difficulty":168.5,"saturation":175.0,"saturationExtrapolated":true,"frontierPct":48.9,"extrapolated":false,"extrapolation":null,"maxTested":169.3,"saturationDate":"2027-02-15","saturationDateLo":"2026-12-13","saturationDateHi":"2027-03-23","saturated":false},{"slug":"orca-bench-hard-rca-fireworks","family":"orca-bench-hard-rca-fireworks","name":"ORCA-bench Hard RCA (Fireworks 324-task subset)","nStates":12,"nModels":12,"difficulty":168.7,"saturation":185.6,"saturationExtrapolated":true,"frontierPct":49.5,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2027-06-02","saturationDateLo":"2026-11-17","saturationDateHi":"2028-12-24","saturated":false},{"slug":"pencil-puzzle-bench-direct-ask-ppbench","family":"pencil-puzzle-bench-direct-ask-ppbench","name":"Pencil Puzzle Bench - Direct Ask","nStates":64,"nModels":49,"difficulty":168.7,"saturation":181.9,"saturationExtrapolated":true,"frontierPct":51.0,"extrapolated":true,"extrapolation":"above","maxTested":163.9,"saturationDate":"2027-06-15","saturationDateLo":"2027-03-03","saturationDateHi":"2027-12-02","saturated":false},{"slug":"ple","family":"ple","name":"PLE (Physical-Engineering's Last Exam)","nStates":14,"nModels":14,"difficulty":168.7,"saturation":187.4,"saturationExtrapolated":true,"frontierPct":45.5,"extrapolated":true,"extrapolation":"above","maxTested":168.0,"saturationDate":"2028-01-03","saturationDateLo":"2027-09-27","saturationDateHi":"2028-03-29","saturated":false},{"slug":"treasuryqa-bench","family":"treasuryqa-bench","name":"TreasuryQA Bench","nStates":10,"nModels":10,"difficulty":168.7,"saturation":201.2,"saturationExtrapolated":true,"frontierPct":49.2,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2028-04-17","saturationDateLo":"2027-04-27","saturationDateHi":"2030-08-23","saturated":false},{"slug":"txbench-od-v1","family":"txbench-od-v1","name":"TxBench Oligo Discovery V0","nStates":13,"nModels":13,"difficulty":168.7,"saturation":205.3,"saturationExtrapolated":true,"frontierPct":49.4,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2028-08-18","saturationDateLo":"2027-10-22","saturationDateHi":"2030-10-08","saturated":false},{"slug":"vibench-2-0","family":"vibench-2-0","name":"ViBench 2.0","nStates":9,"nModels":9,"difficulty":168.7,"saturation":182.5,"saturationExtrapolated":true,"frontierPct":43.2,"extrapolated":true,"extrapolation":"above","maxTested":166.3,"saturationDate":"2027-01-09","saturationDateLo":"2026-09-02","saturationDateHi":"2029-06-16","saturated":false},{"slug":"swe-atlas-codebase-qna-mercor-extended","family":"swe-atlas-codebase-qna-mercor-extended","name":"SWE Atlas Codebase QnA Extended (Mercor held-out)","nStates":33,"nModels":33,"difficulty":168.8,"saturation":181.5,"saturationExtrapolated":true,"frontierPct":43.0,"extrapolated":true,"extrapolation":"above","maxTested":167.9,"saturationDate":"2027-08-13","saturationDateLo":"2027-05-29","saturationDateHi":"2027-11-03","saturated":false},{"slug":"mlcr-aa","family":"mlcr-aa","name":"Medical Long Context Reasoning (MLCR-AA)","nStates":94,"nModels":70,"difficulty":168.9,"saturation":189.6,"saturationExtrapolated":true,"frontierPct":45.5,"extrapolated":true,"extrapolation":"above","maxTested":168.1,"saturationDate":"2028-02-10","saturationDateLo":"2027-10-24","saturationDateHi":"2028-06-23","saturated":false},{"slug":"srebench","family":"srebench","name":"SRE Bench \u2014 Fully Solved","nStates":26,"nModels":26,"difficulty":169.2,"saturation":174.9,"saturationExtrapolated":true,"frontierPct":45.5,"extrapolated":false,"extrapolation":null,"maxTested":169.9,"saturationDate":"2027-02-01","saturationDateLo":"2026-11-24","saturationDateHi":"2027-03-21","saturated":false},{"slug":"biosecbench-function-v1","family":"biosecbench-function-v1","name":"BioSecBench-Function V0","nStates":13,"nModels":13,"difficulty":169.3,"saturation":208.0,"saturationExtrapolated":true,"frontierPct":49.4,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2028-10-27","saturationDateLo":"2027-11-12","saturationDateHi":"2030-12-23","saturated":false},{"slug":"dayjob-data-science","family":"dayjob-data-science","name":"DAYJOB: Data Science","nStates":13,"nModels":13,"difficulty":169.8,"saturation":176.3,"saturationExtrapolated":true,"frontierPct":28.2,"extrapolated":true,"extrapolation":"above","maxTested":168.0,"saturationDate":"2027-04-03","saturationDateLo":"2027-01-25","saturationDateHi":"2027-07-04","saturated":false},{"slug":"infinitebench-en-qa","family":"infinitebench-en-qa","name":"InfiniteBench En.QA","nStates":8,"nModels":8,"difficulty":170.0,"saturation":270.0,"saturationExtrapolated":true,"frontierPct":49.0,"extrapolated":true,"extrapolation":"above","maxTested":130.0,"saturationDate":"2032-11-30","saturationDateLo":"2029-09-17","saturationDateHi":"2037-08-05","saturated":false},{"slug":"snorkel-agentic-coding-2","family":"snorkel-agentic-coding-2","name":"Snorkel Agentic Coding 2.0","nStates":13,"nModels":13,"difficulty":170.0,"saturation":195.2,"saturationExtrapolated":true,"frontierPct":46.1,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2028-03-21","saturationDateLo":"2027-09-02","saturationDateHi":"2029-05-05","saturated":false},{"slug":"deepresearchbench-ii-mercor-130","family":"deepresearchbench-ii","name":"DeepResearch Bench II","nStates":28,"nModels":25,"difficulty":170.1,"saturation":291.1,"saturationExtrapolated":true,"frontierPct":48.8,"extrapolated":true,"extrapolation":"above","maxTested":168.7,"saturationDate":"2033-08-14","saturationDateLo":"2030-08-03","saturationDateHi":"2040-01-26","saturated":false},{"slug":"redlinebench","family":"redlinebench","name":"RedlineBench","nStates":35,"nModels":26,"difficulty":170.1,"saturation":289.9,"saturationExtrapolated":true,"frontierPct":48.9,"extrapolated":true,"extrapolation":"above","maxTested":168.7,"saturationDate":"2033-10-20","saturationDateLo":"2031-03-01","saturationDateHi":"2039-06-19","saturated":false},{"slug":"automationbench-1-0-6-zapier","family":"automationbench-overall-zapier-benchmarks","name":"AutomationBench (Overall)","nStates":112,"nModels":36,"difficulty":170.4,"saturation":183.5,"saturationExtrapolated":true,"frontierPct":40.7,"extrapolated":true,"extrapolation":"above","maxTested":168.8,"saturationDate":"2027-09-10","saturationDateLo":"2027-07-01","saturationDateHi":"2027-11-19","saturated":false},{"slug":"handbook-md-surge-ai-2","family":"handbook-md-surge-ai-2","name":"HANDBOOK.md","nStates":30,"nModels":30,"difficulty":170.4,"saturation":184.6,"saturationExtrapolated":true,"frontierPct":38.8,"extrapolated":true,"extrapolation":"above","maxTested":166.6,"saturationDate":"2027-10-11","saturationDateLo":"2027-07-20","saturationDateHi":"2028-01-23","saturated":false},{"slug":"mmmu-pro-mercor-extended","family":"mmmu-pro-mercor-extended","name":"MMMU-Pro Extended (Mercor held-out)","nStates":37,"nModels":36,"difficulty":170.4,"saturation":199.1,"saturationExtrapolated":true,"frontierPct":52.7,"extrapolated":false,"extrapolation":null,"maxTested":174.4,"saturationDate":"2028-05-14","saturationDateLo":"2028-01-25","saturationDateHi":"2028-09-05","saturated":false},{"slug":"spreadsheetbench-2","family":"spreadsheetbench-2","name":"SpreadsheetBench 2","nStates":15,"nModels":15,"difficulty":170.4,"saturation":198.6,"saturationExtrapolated":true,"frontierPct":44.0,"extrapolated":true,"extrapolation":"above","maxTested":163.3,"saturationDate":"2028-09-07","saturationDateLo":"2028-03-23","saturationDateHi":"2029-05-14","saturated":false},{"slug":"web-clone-bench","family":"web-clone-bench","name":"Web Clone Bench","nStates":35,"nModels":9,"difficulty":170.4,"saturation":175.3,"saturationExtrapolated":true,"frontierPct":18.7,"extrapolated":true,"extrapolation":"above","maxTested":167.9,"saturationDate":"2027-03-20","saturationDateLo":"2027-01-10","saturationDateHi":"2027-05-31","saturated":false},{"slug":"frontier-bench-0-1","family":"frontier-bench-0-1","name":"Frontier-Bench v0.1","nStates":17,"nModels":8,"difficulty":170.5,"saturation":183.9,"saturationExtrapolated":true,"frontierPct":44.7,"extrapolated":true,"extrapolation":"above","maxTested":166.1,"saturationDate":"2027-07-09","saturationDateLo":"2027-03-08","saturationDateHi":"2028-02-05","saturated":false},{"slug":"epibench-benchmarks-bio","family":"epibench-benchmarks-bio","name":"EpiBench","nStates":21,"nModels":21,"difficulty":170.6,"saturation":208.1,"saturationExtrapolated":true,"frontierPct":45.8,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2029-02-12","saturationDateLo":"2028-05-22","saturationDateHi":"2030-03-27","saturated":false},{"slug":"frontiercs-2","family":"frontiercs-2","name":"FrontierCS 2","nStates":10,"nModels":10,"difficulty":170.6,"saturation":185.8,"saturationExtrapolated":true,"frontierPct":40.2,"extrapolated":true,"extrapolation":"above","maxTested":169.0,"saturationDate":"2027-09-02","saturationDateLo":"2027-04-14","saturationDateHi":"2028-07-09","saturated":false},{"slug":"mysterymechanism","family":"mysterymechanism","name":"MysteryMechanism","nStates":27,"nModels":27,"difficulty":170.6,"saturation":190.3,"saturationExtrapolated":true,"frontierPct":43.6,"extrapolated":true,"extrapolation":"above","maxTested":169.1,"saturationDate":"2028-02-03","saturationDateLo":"2027-09-25","saturationDateHi":"2028-08-17","saturated":false},{"slug":"frontiercode-2-0-score-cognition","family":"frontiercode-2-0-score-cognition","name":"FrontierCode 2.0 - Score","nStates":93,"nModels":22,"difficulty":170.7,"saturation":184.9,"saturationExtrapolated":true,"frontierPct":40.1,"extrapolated":true,"extrapolation":"above","maxTested":168.8,"saturationDate":"2027-10-13","saturationDateLo":"2027-08-09","saturationDateHi":"2027-12-14","saturated":false},{"slug":"diligence-bench","family":"diligence-bench","name":"DiligenceBench","nStates":16,"nModels":16,"difficulty":170.8,"saturation":218.8,"saturationExtrapolated":true,"frontierPct":45.6,"extrapolated":true,"extrapolation":"above","maxTested":161.2,"saturationDate":"2029-10-27","saturationDateLo":"2028-07-31","saturationDateHi":"2031-11-12","saturated":false},{"slug":"librarydesignbench","family":"librarydesignbench","name":"LibraryDesignBench","nStates":8,"nModels":8,"difficulty":170.9,"saturation":233.1,"saturationExtrapolated":true,"frontierPct":47.3,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2030-01-21","saturationDateLo":"2028-08-26","saturationDateHi":"2033-09-21","saturated":false},{"slug":"critpt","family":"critpt","name":"CritPt (avg@5)","nStates":586,"nModels":388,"difficulty":171.1,"saturation":185.9,"saturationExtrapolated":true,"frontierPct":40.1,"extrapolated":true,"extrapolation":"above","maxTested":169.3,"saturationDate":"2027-10-31","saturationDateLo":"2027-08-25","saturationDateHi":"2027-12-18","saturated":false},{"slug":"hedge-bench","family":"hedge-bench","name":"Hedge-Bench","nStates":12,"nModels":12,"difficulty":171.1,"saturation":183.4,"saturationExtrapolated":true,"frontierPct":35.5,"extrapolated":true,"extrapolation":"above","maxTested":168.9,"saturationDate":"2027-07-14","saturationDateLo":"2027-03-19","saturationDateHi":"2028-02-26","saturated":false},{"slug":"intphys-2","family":"intphys-2","name":"IntPhys 2","nStates":51,"nModels":51,"difficulty":171.3,"saturation":194.4,"saturationExtrapolated":true,"frontierPct":71.9,"extrapolated":true,"extrapolation":"above","maxTested":168.6,"saturationDate":"2028-04-26","saturationDateLo":"2027-12-09","saturationDateHi":"2028-09-28","saturated":false},{"slug":"arc-agi-3","family":"arc-agi-3","name":"ARC-AGI-3 (Semi-Private)","nStates":75,"nModels":25,"difficulty":171.4,"saturation":181.7,"saturationExtrapolated":true,"frontierPct":31.3,"extrapolated":true,"extrapolation":"above","maxTested":168.0,"saturationDate":"2027-07-15","saturationDateLo":"2027-03-22","saturationDateHi":"2027-12-26","saturated":false},{"slug":"dayjob-finance","family":"dayjob-finance","name":"DAYJOB: Finance","nStates":31,"nModels":31,"difficulty":171.4,"saturation":179.7,"saturationExtrapolated":true,"frontierPct":24.1,"extrapolated":true,"extrapolation":"above","maxTested":167.8,"saturationDate":"2027-07-07","saturationDateLo":"2027-04-22","saturationDateHi":"2027-09-12","saturated":false},{"slug":"agents-last-exam-v1-1","family":"agents-last-exam-v1-1","name":"Agents' Last Exam v1.1 (ALE v1.1)","nStates":21,"nModels":12,"difficulty":171.5,"saturation":200.9,"saturationExtrapolated":true,"frontierPct":43.1,"extrapolated":true,"extrapolation":"above","maxTested":168.9,"saturationDate":"2028-08-31","saturationDateLo":"2028-01-08","saturationDateHi":"2029-11-05","saturated":false},{"slug":"senior-swe-bench","family":"senior-swe-bench","name":"Senior SWE-Bench","nStates":23,"nModels":23,"difficulty":171.6,"saturation":192.6,"saturationExtrapolated":true,"frontierPct":37.8,"extrapolated":true,"extrapolation":"above","maxTested":167.3,"saturationDate":"2028-03-13","saturationDateLo":"2027-09-25","saturationDateHi":"2028-11-29","saturated":false},{"slug":"swe-marathon","family":"swe-marathon","name":"SWE-Marathon v1.0","nStates":15,"nModels":12,"difficulty":171.6,"saturation":186.5,"saturationExtrapolated":true,"frontierPct":33.3,"extrapolated":true,"extrapolation":"above","maxTested":164.9,"saturationDate":"2027-10-16","saturationDateLo":"2027-04-23","saturationDateHi":"2028-10-07","saturated":false},{"slug":"frontiercua","family":"frontiercua","name":"FrontierCUA pass@1","nStates":13,"nModels":13,"difficulty":171.8,"saturation":189.5,"saturationExtrapolated":true,"frontierPct":39.0,"extrapolated":true,"extrapolation":"above","maxTested":168.3,"saturationDate":"2027-11-04","saturationDateLo":"2027-05-06","saturationDateHi":"2028-08-25","saturated":false},{"slug":"agents-last-exam-ale","family":"agents-last-exam-ale","name":"Agents' Last Exam (ALE)","nStates":73,"nModels":30,"difficulty":172.2,"saturation":195.4,"saturationExtrapolated":true,"frontierPct":38.4,"extrapolated":true,"extrapolation":"above","maxTested":168.0,"saturationDate":"2028-07-11","saturationDateLo":"2028-04-14","saturationDateHi":"2028-10-20","saturated":false},{"slug":"zerobench-subquestions-pass-1-zerobench","family":"zerobench-subquestions-pass-1-zerobench","name":"ZeroBench (Subquestions, pass@1)","nStates":64,"nModels":55,"difficulty":172.2,"saturation":239.0,"saturationExtrapolated":true,"frontierPct":47.1,"extrapolated":true,"extrapolation":"above","maxTested":158.0,"saturationDate":"2031-03-06","saturationDateLo":"2030-06-11","saturationDateHi":"2032-02-08","saturated":false},{"slug":"humanitys-sixth-sense","family":"humanitys-sixth-sense","name":"Humanity's Sixth Sense","nStates":28,"nModels":28,"difficulty":172.3,"saturation":204.2,"saturationExtrapolated":true,"frontierPct":42.2,"extrapolated":true,"extrapolation":"above","maxTested":168.4,"saturationDate":"2028-11-30","saturationDateLo":"2028-03-17","saturationDateHi":"2030-05-02","saturated":false},{"slug":"android-bench-2","family":"android-bench-2","name":"Android Bench 2.0","nStates":9,"nModels":9,"difficulty":172.4,"saturation":185.7,"saturationExtrapolated":true,"frontierPct":31.7,"extrapolated":true,"extrapolation":"above","maxTested":168.7,"saturationDate":"2027-10-04","saturationDateLo":"2027-07-01","saturationDateHi":"2028-04-12","saturated":false},{"slug":"txbench-ab-v1-100","family":"txbench-ab-v1-100","name":"TxBench Antibody Discovery V1 (100 evaluations)","nStates":24,"nModels":24,"difficulty":172.4,"saturation":228.6,"saturationExtrapolated":true,"frontierPct":45.0,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2029-11-06","saturationDateLo":"2028-03-20","saturationDateHi":"2033-09-22","saturated":false},{"slug":"legalreasonbench","family":"legalreasonbench","name":"LegalReasonBench","nStates":8,"nModels":8,"difficulty":172.5,"saturation":235.4,"saturationExtrapolated":true,"frontierPct":45.0,"extrapolated":true,"extrapolation":"above","maxTested":167.4,"saturationDate":"2030-04-25","saturationDateLo":"2028-08-02","saturationDateHi":"2034-09-03","saturated":false},{"slug":"exploitgym","family":"exploitgym","name":"ExploitGym","nStates":10,"nModels":10,"difficulty":172.6,"saturation":186.1,"saturationExtrapolated":true,"frontierPct":34.9,"extrapolated":true,"extrapolation":"above","maxTested":166.0,"saturationDate":"2027-07-26","saturationDateLo":"2027-03-05","saturationDateHi":"2028-06-18","saturated":false},{"slug":"pg-llm-50","family":"pg-llm-50","name":"PG-LLM-50","nStates":32,"nModels":16,"difficulty":172.6,"saturation":211.1,"saturationExtrapolated":true,"frontierPct":72.2,"extrapolated":true,"extrapolation":"above","maxTested":165.6,"saturationDate":"2029-06-09","saturationDateLo":"2028-12-07","saturationDateHi":"2030-01-23","saturated":false},{"slug":"activevision","family":"activevision","name":"ActiveVision","nStates":55,"nModels":13,"difficulty":172.9,"saturation":192.2,"saturationExtrapolated":true,"frontierPct":36.1,"extrapolated":true,"extrapolation":"above","maxTested":169.2,"saturationDate":"2028-02-09","saturationDateLo":"2027-08-29","saturationDateHi":"2028-12-12","saturated":false},{"slug":"oncologybench-strict-25-oct9","family":"oncologybench-strict-25-oct9","name":"OncologyBench Strict (25 tasks, October 9 rubric)","nStates":26,"nModels":26,"difficulty":173.1,"saturation":190.4,"saturationExtrapolated":true,"frontierPct":34.9,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2028-02-03","saturationDateLo":"2027-08-25","saturationDateHi":"2028-08-08","saturated":false},{"slug":"astabench-2026-paperfinding","family":"astabench-2026-paperfinding","name":"PaperFindingBench (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":173.2,"saturation":223.0,"saturationExtrapolated":true,"frontierPct":43.6,"extrapolated":true,"extrapolation":"above","maxTested":160.5,"saturationDate":"2030-04-11","saturationDateLo":"2029-07-11","saturationDateHi":"2031-04-04","saturated":false},{"slug":"weirdml-v3","family":"weirdml-v3","name":"WeirdML v3","nStates":24,"nModels":24,"difficulty":173.3,"saturation":185.8,"saturationExtrapolated":true,"frontierPct":31.7,"extrapolated":true,"extrapolation":"above","maxTested":169.2,"saturationDate":"2027-09-27","saturationDateLo":"2027-07-07","saturationDateHi":"2028-01-16","saturated":false},{"slug":"dayjob-healthcare","family":"dayjob-healthcare","name":"DAYJOB: Healthcare","nStates":31,"nModels":31,"difficulty":173.5,"saturation":181.5,"saturationExtrapolated":true,"frontierPct":14.6,"extrapolated":true,"extrapolation":"above","maxTested":167.8,"saturationDate":"2027-08-09","saturationDateLo":"2027-05-07","saturationDateHi":"2027-12-03","saturated":false},{"slug":"sudo-l7-surge-ai","family":"sudo-l7-surge-ai","name":"sudo L7","nStates":13,"nModels":13,"difficulty":173.6,"saturation":190.1,"saturationExtrapolated":true,"frontierPct":29.4,"extrapolated":true,"extrapolation":"above","maxTested":168.4,"saturationDate":"2028-01-09","saturationDateLo":"2027-08-21","saturationDateHi":"2028-10-23","saturated":false},{"slug":"real-swe","family":"real-swe","name":"Real-SWE","nStates":8,"nModels":8,"difficulty":174.1,"saturation":217.8,"saturationExtrapolated":true,"frontierPct":40.6,"extrapolated":true,"extrapolation":"above","maxTested":167.7,"saturationDate":"2029-05-28","saturationDateLo":"2028-01-13","saturationDateHi":"2033-03-27","saturated":false},{"slug":"rwe-bench","family":"rwe-bench","name":"RWE-bench","nStates":9,"nModels":9,"difficulty":174.2,"saturation":179.5,"saturationExtrapolated":true,"frontierPct":12.7,"extrapolated":true,"extrapolation":"above","maxTested":172.5,"saturationDate":"2027-02-18","saturationDateLo":"2026-12-02","saturationDateHi":"2027-07-16","saturated":false},{"slug":"terminal-bench-2-1-mercor-extended","family":"terminal-bench-2-1-mercor-extended","name":"Terminal-Bench 2.1 Extended (Mercor held-out)","nStates":44,"nModels":41,"difficulty":174.3,"saturation":214.3,"saturationExtrapolated":true,"frontierPct":42.2,"extrapolated":true,"extrapolation":"above","maxTested":169.7,"saturationDate":"2029-07-16","saturationDateLo":"2028-11-27","saturationDateHi":"2030-08-02","saturated":false},{"slug":"deepsecbench-revalidated-v1-233","family":"deepsecbench","name":"DeepsecBench","nStates":54,"nModels":28,"difficulty":174.4,"saturation":195.1,"saturationExtrapolated":true,"frontierPct":35.3,"extrapolated":true,"extrapolation":"above","maxTested":169.7,"saturationDate":"2028-05-01","saturationDateLo":"2027-12-21","saturationDateHi":"2028-10-19","saturated":false},{"slug":"frontierphysics-0-1","family":"frontierphysics-0-1","name":"FrontierPhysics v0.1","nStates":22,"nModels":18,"difficulty":174.5,"saturation":183.0,"saturationExtrapolated":true,"frontierPct":15.7,"extrapolated":true,"extrapolation":"above","maxTested":169.0,"saturationDate":"2027-08-23","saturationDateLo":"2027-05-28","saturationDateHi":"2027-12-01","saturated":false},{"slug":"dayjob-accounting","family":"dayjob-accounting","name":"DAYJOB: Accounting","nStates":13,"nModels":13,"difficulty":174.7,"saturation":190.7,"saturationExtrapolated":true,"frontierPct":25.7,"extrapolated":true,"extrapolation":"above","maxTested":168.7,"saturationDate":"2027-12-31","saturationDateLo":"2027-07-20","saturationDateHi":"2029-03-20","saturated":false},{"slug":"snorkel-swe-bench-cli","family":"snorkel-swe-bench-cli","name":"Snorkel SWE-bench CLI","nStates":13,"nModels":13,"difficulty":174.9,"saturation":185.6,"saturationExtrapolated":true,"frontierPct":19.5,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2027-10-05","saturationDateLo":"2027-05-26","saturationDateHi":"2028-04-28","saturated":false},{"slug":"taxcalcbench-ty25","family":"taxcalcbench-ty25","name":"TaxCalcBench TY25","nStates":30,"nModels":26,"difficulty":174.9,"saturation":189.3,"saturationExtrapolated":true,"frontierPct":34.6,"extrapolated":true,"extrapolation":"above","maxTested":173.0,"saturationDate":"2027-10-20","saturationDateLo":"2027-07-06","saturationDateHi":"2028-03-19","saturated":false},{"slug":"scbench-long-v1-verified","family":"scbench-long-v1-verified","name":"ScBench-Long V0 Verified","nStates":15,"nModels":15,"difficulty":175.0,"saturation":207.0,"saturationExtrapolated":true,"frontierPct":36.2,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2028-11-17","saturationDateLo":"2027-09-27","saturationDateHi":"2031-03-01","saturated":false},{"slug":"vials","family":"vials","name":"VIALS","nStates":20,"nModels":20,"difficulty":175.2,"saturation":207.9,"saturationExtrapolated":true,"frontierPct":37.5,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2029-01-22","saturationDateLo":"2028-04-02","saturationDateHi":"2030-06-16","saturated":false},{"slug":"interactivebench","family":"interactivebench","name":"InteractiveBench (task pass)","nStates":10,"nModels":10,"difficulty":175.5,"saturation":213.7,"saturationExtrapolated":true,"frontierPct":38.7,"extrapolated":true,"extrapolation":"above","maxTested":166.1,"saturationDate":"2029-03-30","saturationDateLo":"2028-05-10","saturationDateHi":"2031-03-28","saturated":false},{"slug":"vals-rsi-index-v1-1-four-tasks","family":"vals-rsi-index-v1-1-four-tasks","name":"Vals RSI Index v1.1 (four tasks)","nStates":23,"nModels":23,"difficulty":175.6,"saturation":200.6,"saturationExtrapolated":true,"frontierPct":31.9,"extrapolated":true,"extrapolation":"above","maxTested":167.9,"saturationDate":"2028-10-26","saturationDateLo":"2028-06-07","saturationDateHi":"2029-05-07","saturated":false},{"slug":"gdp-pdf-all-pass-artificial-analysis","family":"gdp-pdf-all-pass-artificial-analysis","name":"GDP.pdf (Artificial Analysis, All-pass)","nStates":228,"nModels":146,"difficulty":175.8,"saturation":198.9,"saturationExtrapolated":true,"frontierPct":32.3,"extrapolated":true,"extrapolation":"above","maxTested":168.6,"saturationDate":"2028-09-20","saturationDateLo":"2028-07-07","saturationDateHi":"2028-12-07","saturated":false},{"slug":"witnessbench-2-0","family":"witnessbench-2-0","name":"WitnessBench v2.0","nStates":12,"nModels":12,"difficulty":175.8,"saturation":192.5,"saturationExtrapolated":true,"frontierPct":24.4,"extrapolated":true,"extrapolation":"above","maxTested":169.8,"saturationDate":"2028-02-22","saturationDateLo":"2027-10-05","saturationDateHi":"2028-11-16","saturated":false},{"slug":"remote-labor-index","family":"remote-labor-index","name":"Remote Labor Index (RLI)","nStates":17,"nModels":16,"difficulty":175.9,"saturation":190.2,"saturationExtrapolated":true,"frontierPct":21.3,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2028-02-20","saturationDateLo":"2027-11-24","saturationDateHi":"2028-06-05","saturated":false},{"slug":"gdp-pdf","family":"gdp-pdf","name":"GDP.pdf","nStates":47,"nModels":39,"difficulty":176.0,"saturation":203.0,"saturationExtrapolated":true,"frontierPct":34.1,"extrapolated":true,"extrapolation":"above","maxTested":168.7,"saturationDate":"2028-12-20","saturationDateLo":"2028-08-27","saturationDateHi":"2029-04-15","saturated":false},{"slug":"continual-learning-exam","family":"continual-learning-exam","name":"Continual Learning Exam (Trial 1)","nStates":12,"nModels":12,"difficulty":176.4,"saturation":211.0,"saturationExtrapolated":true,"frontierPct":36.0,"extrapolated":true,"extrapolation":"above","maxTested":164.6,"saturationDate":"2029-02-08","saturationDateLo":"2028-02-25","saturationDateHi":"2031-01-09","saturated":false},{"slug":"vcb-1-100","family":"vcb-1-100","name":"Vibe Code Bench 1\u2013100","nStates":23,"nModels":23,"difficulty":176.5,"saturation":199.9,"saturationExtrapolated":true,"frontierPct":29.0,"extrapolated":true,"extrapolation":"above","maxTested":167.0,"saturationDate":"2028-09-25","saturationDateLo":"2028-04-10","saturationDateHi":"2029-05-31","saturated":false},{"slug":"witnessbench-1-0","family":"witnessbench-1-0","name":"WitnessBench v1.0","nStates":17,"nModels":17,"difficulty":177.3,"saturation":198.4,"saturationExtrapolated":true,"frontierPct":27.2,"extrapolated":true,"extrapolation":"above","maxTested":170.4,"saturationDate":"2028-07-22","saturationDateLo":"2028-01-18","saturationDateHi":"2029-04-26","saturated":false},{"slug":"cl-bench-life","family":"cl-bench-life","name":"CL-bench Life (Overall)","nStates":28,"nModels":15,"difficulty":177.4,"saturation":208.0,"saturationExtrapolated":true,"frontierPct":32.4,"extrapolated":true,"extrapolation":"above","maxTested":159.7,"saturationDate":"2029-03-16","saturationDateLo":"2028-09-07","saturationDateHi":"2029-12-30","saturated":false},{"slug":"opends-world","family":"opends-world","name":"OpenDS-World","nStates":9,"nModels":9,"difficulty":177.4,"saturation":212.1,"saturationExtrapolated":true,"frontierPct":32.9,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2028-12-21","saturationDateLo":"2027-11-06","saturationDateHi":"2031-07-08","saturated":false},{"slug":"zerobench-main-mean-pass-1-five-samples","family":"zerobench-main-questions-pass-1-zerobench","name":"ZeroBench (Main questions, pass@1)","nStates":54,"nModels":49,"difficulty":177.5,"saturation":198.0,"saturationExtrapolated":true,"frontierPct":29.2,"extrapolated":true,"extrapolation":"above","maxTested":170.8,"saturationDate":"2028-07-11","saturationDateLo":"2028-04-08","saturationDateHi":"2028-10-24","saturated":false},{"slug":"quant-arena","family":"quant-arena","name":"Quant-Arena (iterative total score)","nStates":38,"nModels":9,"difficulty":177.7,"saturation":275.6,"saturationExtrapolated":true,"frontierPct":45.1,"extrapolated":true,"extrapolation":"above","maxTested":171.3,"saturationDate":"2033-06-12","saturationDateLo":"2031-03-01","saturationDateHi":"2037-01-29","saturated":false},{"slug":"agentvqa","family":"agentvqa","name":"AgentVQA","nStates":18,"nModels":16,"difficulty":177.8,"saturation":244.3,"saturationExtrapolated":true,"frontierPct":57.9,"extrapolated":true,"extrapolation":"above","maxTested":163.8,"saturationDate":"2031-03-06","saturationDateLo":"2029-08-06","saturationDateHi":"2033-07-05","saturated":false},{"slug":"atlas-finance-1-1","family":"atlas-finance-1-1","name":"ATLAS-Finance v1.1","nStates":10,"nModels":10,"difficulty":178.6,"saturation":191.1,"saturationExtrapolated":true,"frontierPct":10.6,"extrapolated":true,"extrapolation":"above","maxTested":166.5,"saturationDate":"2028-02-06","saturationDateLo":"2027-10-01","saturationDateHi":"2028-10-19","saturated":false},{"slug":"gdp-xlsx","family":"gdp-xlsx","name":"GDP.xlsx","nStates":16,"nModels":16,"difficulty":178.9,"saturation":205.6,"saturationExtrapolated":true,"frontierPct":26.5,"extrapolated":true,"extrapolation":"above","maxTested":168.1,"saturationDate":"2029-02-18","saturationDateLo":"2028-06-09","saturationDateHi":"2030-02-14","saturated":false},{"slug":"hieroglyphbench","family":"hieroglyphbench","name":"HieroglyphBench","nStates":30,"nModels":30,"difficulty":179.8,"saturation":233.0,"saturationExtrapolated":true,"frontierPct":41.6,"extrapolated":true,"extrapolation":"above","maxTested":169.6,"saturationDate":"2030-01-18","saturationDateLo":"2028-08-19","saturationDateHi":"2032-08-25","saturated":false},{"slug":"atlas-finance","family":"atlas-finance","name":"ATLAS-Finance v1.0","nStates":10,"nModels":10,"difficulty":180.2,"saturation":194.7,"saturationExtrapolated":true,"frontierPct":11.3,"extrapolated":true,"extrapolation":"above","maxTested":167.4,"saturationDate":"2028-04-21","saturationDateLo":"2027-10-14","saturationDateHi":"2029-03-22","saturated":false},{"slug":"spatialbench-long-v1-verified","family":"spatialbench-long-v1-verified","name":"SpatialBench-Long V0 Verified","nStates":14,"nModels":14,"difficulty":180.7,"saturation":228.8,"saturationExtrapolated":true,"frontierPct":34.4,"extrapolated":true,"extrapolation":"above","maxTested":165.5,"saturationDate":"2030-03-30","saturationDateLo":"2028-10-12","saturationDateHi":"2033-03-11","saturated":false},{"slug":"txbench-od-v1-113","family":"txbench-od-v1-113","name":"TxBench Oligo Discovery V1 (113 evaluations)","nStates":24,"nModels":24,"difficulty":180.8,"saturation":255.8,"saturationExtrapolated":true,"frontierPct":39.1,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2031-11-08","saturationDateLo":"2029-06-08","saturationDateHi":"2035-09-04","saturated":false},{"slug":"dayjob-law","family":"dayjob-law","name":"DAYJOB: Law","nStates":13,"nModels":13,"difficulty":181.0,"saturation":206.8,"saturationExtrapolated":true,"frontierPct":22.3,"extrapolated":true,"extrapolation":"above","maxTested":169.7,"saturationDate":"2028-11-23","saturationDateLo":"2028-01-06","saturationDateHi":"2030-08-22","saturated":false},{"slug":"globebench","family":"globebench","name":"GlobeBench","nStates":17,"nModels":17,"difficulty":181.1,"saturation":213.0,"saturationExtrapolated":true,"frontierPct":27.1,"extrapolated":true,"extrapolation":"above","maxTested":168.1,"saturationDate":"2029-05-22","saturationDateLo":"2028-05-03","saturationDateHi":"2031-05-01","saturated":false},{"slug":"mlcr-aa-2026-10","family":"mlcr-aa-2026-10","name":"Medical Long Context Reasoning (MLCR-AA, October 2026 methodology)","nStates":171,"nModels":99,"difficulty":181.3,"saturation":211.9,"saturationExtrapolated":true,"frontierPct":27.3,"extrapolated":true,"extrapolation":"above","maxTested":168.4,"saturationDate":"2029-08-02","saturationDateLo":"2029-03-14","saturationDateHi":"2030-01-13","saturated":false},{"slug":"astabench-2026-discoverybench","family":"astabench-2026-discoverybench","name":"DiscoveryBench (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":181.4,"saturation":267.0,"saturationExtrapolated":true,"frontierPct":41.3,"extrapolated":true,"extrapolation":"above","maxTested":161.9,"saturationDate":"2032-12-21","saturationDateLo":"2031-03-04","saturationDateHi":"2035-06-05","saturated":false},{"slug":"binaryaudit","family":"binaryaudit","name":"BinaryAudit (Binary Analysis)","nStates":26,"nModels":20,"difficulty":182.0,"saturation":237.8,"saturationExtrapolated":true,"frontierPct":34.8,"extrapolated":true,"extrapolation":"above","maxTested":159.0,"saturationDate":"2030-11-17","saturationDateLo":"2028-11-24","saturationDateHi":"2033-11-06","saturated":false},{"slug":"epibench-v2","family":"epibench-v2","name":"EpiBench V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":182.4,"saturation":221.2,"saturationExtrapolated":true,"frontierPct":29.9,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2029-12-20","saturationDateLo":"2028-12-27","saturationDateHi":"2031-05-07","saturated":false},{"slug":"build-a-chip-bench-20-tasks","family":"build-a-chip-bench","name":"Build-A-Chip-Bench","nStates":8,"nModels":8,"difficulty":183.8,"saturation":201.4,"saturationExtrapolated":true,"frontierPct":6.0,"extrapolated":true,"extrapolation":"above","maxTested":166.1,"saturationDate":"2028-05-25","saturationDateLo":"2027-04-14","saturationDateHi":"2030-08-31","saturated":false},{"slug":"dayjob-tax","family":"dayjob-tax","name":"DAYJOB: Tax","nStates":14,"nModels":14,"difficulty":184.1,"saturation":198.2,"saturationExtrapolated":true,"frontierPct":5.0,"extrapolated":true,"extrapolation":"above","maxTested":168.5,"saturationDate":"2028-06-04","saturationDateLo":"2027-09-27","saturationDateHi":"2029-09-14","saturated":false},{"slug":"programbench","family":"programbench","name":"ProgramBench","nStates":60,"nModels":58,"difficulty":184.7,"saturation":196.4,"saturationExtrapolated":true,"frontierPct":3.5,"extrapolated":true,"extrapolation":"above","maxTested":167.8,"saturationDate":"2028-06-22","saturationDateLo":"2028-02-03","saturationDateHi":"2029-03-19","saturated":false},{"slug":"gdpx","family":"gdpx","name":"GDPx","nStates":17,"nModels":17,"difficulty":185.1,"saturation":201.9,"saturationExtrapolated":true,"frontierPct":9.2,"extrapolated":true,"extrapolation":"above","maxTested":168.8,"saturationDate":"2028-10-20","saturationDateLo":"2028-05-26","saturationDateHi":"2029-07-26","saturated":false},{"slug":"snorkel-legal","family":"snorkel-legal","name":"SnorkelLegal","nStates":13,"nModels":13,"difficulty":185.4,"saturation":255.8,"saturationExtrapolated":true,"frontierPct":34.4,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2031-08-21","saturationDateLo":"2029-05-19","saturationDateHi":"2036-08-21","saturated":false},{"slug":"ebr-bench-v1","family":"ebr-bench-v1","name":"EBR-bench v1","nStates":12,"nModels":12,"difficulty":185.5,"saturation":220.0,"saturationExtrapolated":true,"frontierPct":23.9,"extrapolated":true,"extrapolation":"above","maxTested":161.3,"saturationDate":"2029-07-21","saturationDateLo":"2028-04-13","saturationDateHi":"2031-11-27","saturated":false},{"slug":"spatialbench-long-v2","family":"spatialbench-long-v2","name":"SpatialBench-Long V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":186.1,"saturation":244.8,"saturationExtrapolated":true,"frontierPct":32.3,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2031-04-03","saturationDateLo":"2029-07-30","saturationDateHi":"2033-12-17","saturated":false},{"slug":"harvey-lab-mercor","family":"harvey-lab-mercor","name":"Harvey LAB (Mercor run, 1,749 public tasks)","nStates":18,"nModels":18,"difficulty":186.7,"saturation":210.5,"saturationExtrapolated":true,"frontierPct":11.6,"extrapolated":true,"extrapolation":"above","maxTested":165.0,"saturationDate":"2029-03-09","saturationDateLo":"2028-02-06","saturationDateHi":"2031-03-16","saturated":false},{"slug":"scbench-long-v2","family":"scbench-long-v2","name":"ScBench-Long V1 Clean Reruns","nStates":24,"nModels":24,"difficulty":187.3,"saturation":225.1,"saturationExtrapolated":true,"frontierPct":22.7,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2030-03-25","saturationDateLo":"2028-11-17","saturationDateHi":"2032-09-21","saturated":false},{"slug":"radle-2","family":"radle-2","name":"Radiology's Last Exam 2.0 (RadLE-C)","nStates":9,"nModels":9,"difficulty":187.6,"saturation":267.6,"saturationExtrapolated":true,"frontierPct":36.4,"extrapolated":true,"extrapolation":"above","maxTested":166.1,"saturationDate":"2032-11-04","saturationDateLo":"2030-12-27","saturationDateHi":"2035-10-07","saturated":false},{"slug":"astabench-2026-arxivdigestables","family":"astabench-2026-arxivdigestables","name":"ArxivDIGESTables-Clean (AstaBench 2026)","nStates":13,"nModels":13,"difficulty":188.6,"saturation":289.0,"saturationExtrapolated":true,"frontierPct":38.3,"extrapolated":true,"extrapolation":"above","maxTested":160.7,"saturationDate":"2034-04-30","saturationDateLo":"2032-04-12","saturationDateHi":"2037-03-20","saturated":false},{"slug":"cl-bench","family":"cl-bench","name":"CL-bench (Overall)","nStates":29,"nModels":23,"difficulty":188.6,"saturation":245.9,"saturationExtrapolated":true,"frontierPct":30.0,"extrapolated":true,"extrapolation":"above","maxTested":157.9,"saturationDate":"2031-10-09","saturationDateLo":"2030-03-31","saturationDateHi":"2033-08-16","saturated":false},{"slug":"hlab","family":"hlab","name":"Harvey's Legal Agent Benchmark","nStates":79,"nModels":74,"difficulty":188.6,"saturation":214.6,"saturationExtrapolated":true,"frontierPct":13.4,"extrapolated":true,"extrapolation":"above","maxTested":167.8,"saturationDate":"2029-09-27","saturationDateLo":"2029-01-11","saturationDateHi":"2030-09-27","saturated":false},{"slug":"cua-bench","family":"cua-bench","name":"CUA-bench","nStates":9,"nModels":9,"difficulty":189.1,"saturation":211.9,"saturationExtrapolated":true,"frontierPct":8.8,"extrapolated":true,"extrapolation":"above","maxTested":169.3,"saturationDate":"2029-03-04","saturationDateLo":"2028-03-04","saturationDateHi":"2030-12-25","saturated":false},{"slug":"swe-shift-shifted","family":"swe-shift-shifted","name":"SWE-Shift (Shifted)","nStates":16,"nModels":16,"difficulty":189.4,"saturation":221.4,"saturationExtrapolated":true,"frontierPct":18.8,"extrapolated":true,"extrapolation":"above","maxTested":168.7,"saturationDate":"2029-12-27","saturationDateLo":"2029-03-30","saturationDateHi":"2031-01-12","saturated":false},{"slug":"frontiermath-erdos","family":"frontiermath-erdos","name":"FrontierMath Erd\u0151s","nStates":14,"nModels":14,"difficulty":190.0,"saturation":203.2,"saturationExtrapolated":true,"frontierPct":1.2,"extrapolated":true,"extrapolation":"above","maxTested":169.9,"saturationDate":"2028-09-13","saturationDateLo":"2027-10-17","saturationDateHi":"2030-05-24","saturated":false},{"slug":"spatialbench-long-benchmarks-bio","family":"spatialbench-long-benchmarks-bio","name":"SpatialBench-Long","nStates":12,"nModels":12,"difficulty":190.6,"saturation":219.3,"saturationExtrapolated":true,"frontierPct":13.6,"extrapolated":true,"extrapolation":"above","maxTested":160.0,"saturationDate":"2029-11-28","saturationDateLo":"2028-07-20","saturationDateHi":"2031-10-15","saturated":false},{"slug":"snorkel-finance-2","family":"snorkel-finance-2","name":"SnorkelFinance 2.0","nStates":13,"nModels":13,"difficulty":193.6,"saturation":233.3,"saturationExtrapolated":true,"frontierPct":16.4,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2030-08-27","saturationDateLo":"2028-11-26","saturationDateHi":"2033-09-04","saturated":false},{"slug":"dual-arm-sim","family":"dual-arm-sim","name":"Dual-Arm Sim (task completion)","nStates":10,"nModels":10,"difficulty":194.2,"saturation":215.7,"saturationExtrapolated":true,"frontierPct":3.4,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2029-06-23","saturationDateLo":"2028-02-14","saturationDateHi":"2032-01-05","saturated":false},{"slug":"snorkel-manufacturing","family":"snorkel-manufacturing","name":"SnorkelManufacturing","nStates":13,"nModels":13,"difficulty":195.1,"saturation":233.3,"saturationExtrapolated":true,"frontierPct":14.8,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2030-07-30","saturationDateLo":"2028-12-09","saturationDateHi":"2033-01-09","saturated":false},{"slug":"pharmabench-micro1","family":"pharmabench-micro1","name":"micro1 PharmaBench","nStates":12,"nModels":12,"difficulty":195.7,"saturation":233.0,"saturationExtrapolated":true,"frontierPct":13.6,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2030-07-24","saturationDateLo":"2028-12-19","saturationDateHi":"2033-04-17","saturated":false},{"slug":"snorkel-revops","family":"snorkel-revops","name":"SnorkelRevOps","nStates":13,"nModels":13,"difficulty":195.9,"saturation":233.6,"saturationExtrapolated":true,"frontierPct":14.3,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2030-09-19","saturationDateLo":"2029-01-26","saturationDateHi":"2032-12-28","saturated":false},{"slug":"snorkel-underwrite-2","family":"snorkel-underwrite-2","name":"SnorkelUnderwrite 2.0","nStates":13,"nModels":13,"difficulty":196.1,"saturation":257.3,"saturationExtrapolated":true,"frontierPct":24.3,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2032-03-29","saturationDateLo":"2029-10-30","saturationDateHi":"2035-04-15","saturated":false},{"slug":"swe-sweep","family":"swe-sweep","name":"SWE-sweep","nStates":14,"nModels":11,"difficulty":196.8,"saturation":217.5,"saturationExtrapolated":true,"frontierPct":4.9,"extrapolated":true,"extrapolation":"above","maxTested":172.2,"saturationDate":"2029-07-14","saturationDateLo":"2028-10-15","saturationDateHi":"2030-09-09","saturated":false},{"slug":"snorkel-workplace-agents","family":"snorkel-workplace-agents","name":"Snorkel WorkplaceAgents","nStates":12,"nModels":12,"difficulty":198.2,"saturation":242.5,"saturationExtrapolated":true,"frontierPct":16.5,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2031-03-04","saturationDateLo":"2029-08-06","saturationDateHi":"2033-06-11","saturated":false},{"slug":"hlab-contracts","family":"hlab-contracts","name":"Harvey LAB: In-House Contracting (held-out)","nStates":13,"nModels":13,"difficulty":199.4,"saturation":225.8,"saturationExtrapolated":true,"frontierPct":5.1,"extrapolated":true,"extrapolation":"above","maxTested":167.2,"saturationDate":"2030-03-13","saturationDateLo":"2028-11-06","saturationDateHi":"2032-06-06","saturated":false},{"slug":"hlab-v1-1","family":"hlab-v1-1","name":"HLAB-Vals-1.1","nStates":32,"nModels":32,"difficulty":201.3,"saturation":245.7,"saturationExtrapolated":true,"frontierPct":13.8,"extrapolated":true,"extrapolation":"above","maxTested":168.1,"saturationDate":"2031-07-13","saturationDateLo":"2029-12-15","saturationDateHi":"2033-10-08","saturated":false},{"slug":"forecastbench-tournament-overall-brier-index","family":"forecastbench-tournament-overall-brier-index","name":"ForecastBench Tournament Overall Brier Index","nStates":96,"nModels":88,"difficulty":209.7,"saturation":344.1,"saturationExtrapolated":true,"frontierPct":67.1,"extrapolated":true,"extrapolation":"above","maxTested":167.4,"saturationDate":"2037-12-03","saturationDateLo":"2036-05-03","saturationDateHi":"2039-10-03","saturated":false},{"slug":"scipredict-scale-ai-2","family":"scipredict-scale-ai-2","name":"SciPredict","nStates":15,"nModels":15,"difficulty":218.2,"saturation":338.2,"saturationExtrapolated":true,"frontierPct":28.0,"extrapolated":true,"extrapolation":"above","maxTested":154.2,"saturationDate":"2037-10-17","saturationDateLo":"2035-01-17","saturationDateHi":"2040-08-15","saturated":false},{"slug":"forecastbench-baseline-overall-brier-index","family":"forecastbench-baseline-overall-brier-index","name":"ForecastBench Baseline Overall Brier Index","nStates":81,"nModels":77,"difficulty":218.5,"saturation":317.5,"saturationExtrapolated":true,"frontierPct":62.3,"extrapolated":true,"extrapolation":"above","maxTested":160.5,"saturationDate":"2036-04-28","saturationDateLo":"2034-06-10","saturationDateHi":"2038-04-25","saturated":false}]},"diagnostics":{"reasoning":{"scale":"latent_phi","scorePointScale":25.172,"effects":[{"condition":"default (reasoning always on)","level":"default (reasoning always on)","ordinal_rank":null,"config_count":80,"mean":0.0954,"ci_low":0.0809,"ci_high":0.1115,"score_points_mean":2.401,"score_points_ci_low":2.037,"score_points_ci_high":2.806,"p_gt_0":1.0},{"condition":"effort:heavy","level":"effort:heavy","ordinal_rank":null,"config_count":3,"mean":0.0887,"ci_low":-0.0382,"ci_high":0.2053,"score_points_mean":2.232,"score_points_ci_low":-0.96,"score_points_ci_high":5.168,"p_gt_0":0.9244},{"condition":"effort:extended","level":"effort:extended","ordinal_rank":null,"config_count":8,"mean":0.0764,"ci_low":0.0035,"ci_high":0.1452,"score_points_mean":1.924,"score_points_ci_low":0.089,"score_points_ci_high":3.654,"p_gt_0":0.9794},{"condition":"effort:2k","level":"effort:2k","ordinal_rank":null,"config_count":5,"mean":0.0592,"ci_low":-0.068,"ci_high":0.1994,"score_points_mean":1.491,"score_points_ci_low":-1.712,"score_points_ci_high":5.019,"p_gt_0":0.8169},{"condition":"effort:0.99","level":"effort:0.99","ordinal_rank":null,"config_count":3,"mean":0.0524,"ci_low":-0.0117,"ci_high":0.1121,"score_points_mean":1.32,"score_points_ci_low":-0.294,"score_points_ci_high":2.821,"p_gt_0":0.9542},{"condition":"enabled","level":"enabled","ordinal_rank":null,"config_count":207,"mean":0.0424,"ci_low":0.0282,"ci_high":0.0566,"score_points_mean":1.067,"score_points_ci_low":0.709,"score_points_ci_high":1.424,"p_gt_0":1.0},{"condition":"effort:max","level":"effort:max","ordinal_rank":5,"config_count":51,"mean":0.0341,"ci_low":0.0178,"ci_high":0.0508,"score_points_mean":0.858,"score_points_ci_low":0.448,"score_points_ci_high":1.279,"p_gt_0":1.0},{"condition":"effort:xhigh","level":"effort:xhigh","ordinal_rank":4,"config_count":82,"mean":0.0303,"ci_low":0.013,"ci_high":0.0476,"score_points_mean":0.764,"score_points_ci_low":0.327,"score_points_ci_high":1.198,"p_gt_0":0.9997},{"condition":"effort:high","level":"effort:high","ordinal_rank":3,"config_count":163,"mean":0.0204,"ci_low":0.0088,"ci_high":0.0324,"score_points_mean":0.513,"score_points_ci_low":0.222,"score_points_ci_high":0.815,"p_gt_0":0.9994},{"condition":"effort:16k","level":"effort:16k","ordinal_rank":null,"config_count":17,"mean":0.0098,"ci_low":-0.0167,"ci_high":0.0351,"score_points_mean":0.247,"score_points_ci_low":-0.42,"score_points_ci_high":0.885,"p_gt_0":0.7789},{"condition":"effort:65535","level":"effort:65535","ordinal_rank":null,"config_count":3,"mean":0.0046,"ci_low":-0.1318,"ci_high":0.1344,"score_points_mean":0.116,"score_points_ci_low":-3.318,"score_points_ci_high":3.384,"p_gt_0":0.5281},{"condition":"effort:128k","level":"effort:128k","ordinal_rank":null,"config_count":3,"mean":0.0005,"ci_low":-0.0883,"ci_high":0.0978,"score_points_mean":0.014,"score_points_ci_low":-2.224,"score_points_ci_high":2.462,"p_gt_0":0.4958},{"condition":"default","level":"default","ordinal_rank":null,"config_count":764,"mean":0.0,"ci_low":0.0,"ci_high":0.0,"score_points_mean":0.0,"score_points_ci_low":0.0,"score_points_ci_high":0.0,"p_gt_0":0.0},{"condition":"effort:32k","level":"effort:32k","ordinal_rank":null,"config_count":10,"mean":-0.008,"ci_low":-0.0418,"ci_high":0.0221,"score_points_mean":-0.202,"score_points_ci_low":-1.052,"score_points_ci_high":0.557,"p_gt_0":0.3114},{"condition":"effort:60000","level":"effort:60000","ordinal_rank":null,"config_count":3,"mean":-0.0097,"ci_low":-0.1537,"ci_high":0.1171,"score_points_mean":-0.245,"score_points_ci_low":-3.868,"score_points_ci_high":2.946,"p_gt_0":0.4561},{"condition":"effort:8k","level":"effort:8k","ordinal_rank":null,"config_count":11,"mean":-0.0144,"ci_low":-0.0449,"ci_high":0.0151,"score_points_mean":-0.363,"score_points_ci_low":-1.129,"score_points_ci_high":0.38,"p_gt_0":0.1706},{"condition":"effort:medium","level":"effort:medium","ordinal_rank":2,"config_count":105,"mean":-0.0148,"ci_low":-0.0352,"ci_high":0.0049,"score_points_mean":-0.372,"score_points_ci_low":-0.886,"score_points_ci_high":0.123,"p_gt_0":0.0686},{"condition":"effort:adaptive","level":"effort:adaptive","ordinal_rank":null,"config_count":3,"mean":-0.0163,"ci_low":-0.1572,"ci_high":0.1297,"score_points_mean":-0.411,"score_points_ci_low":-3.956,"score_points_ci_high":3.265,"p_gt_0":0.4058},{"condition":"effort:auto","level":"effort:auto","ordinal_rank":null,"config_count":3,"mean":-0.0167,"ci_low":-0.163,"ci_high":0.1285,"score_points_mean":-0.42,"score_points_ci_low":-4.104,"score_points_ci_high":3.234,"p_gt_0":0.4011},{"condition":"effort:64k","level":"effort:64k","ordinal_rank":null,"config_count":10,"mean":-0.0326,"ci_low":-0.065,"ci_high":0.0011,"score_points_mean":-0.819,"score_points_ci_low":-1.636,"score_points_ci_high":0.027,"p_gt_0":0.0294},{"condition":"effort:59k","level":"effort:59k","ordinal_rank":null,"config_count":4,"mean":-0.0389,"ci_low":-0.1177,"ci_high":0.0355,"score_points_mean":-0.979,"score_points_ci_low":-2.963,"score_points_ci_high":0.893,"p_gt_0":0.1533},{"condition":"disabled:max","level":"disabled:max","ordinal_rank":null,"config_count":3,"mean":-0.0403,"ci_low":-0.1172,"ci_high":0.0438,"score_points_mean":-1.014,"score_points_ci_low":-2.95,"score_points_ci_high":1.104,"p_gt_0":0.1575},{"condition":"effort:31488","level":"effort:31488","ordinal_rank":null,"config_count":5,"mean":-0.0505,"ci_low":-0.2153,"ci_high":0.1118,"score_points_mean":-1.27,"score_points_ci_low":-5.418,"score_points_ci_high":2.815,"p_gt_0":0.265},{"condition":"effort:1k","level":"effort:1k","ordinal_rank":null,"config_count":9,"mean":-0.0675,"ci_low":-0.1031,"ci_high":-0.0318,"score_points_mean":-1.7,"score_points_ci_low":-2.596,"score_points_ci_high":-0.801,"p_gt_0":0.0008},{"condition":"effort:low","level":"effort:low","ordinal_rank":1,"config_count":110,"mean":-0.0862,"ci_low":-0.1077,"ci_high":-0.0642,"score_points_mean":-2.171,"score_points_ci_low":-2.711,"score_points_ci_high":-1.617,"p_gt_0":0.0},{"condition":"disabled:low","level":"disabled:low","ordinal_rank":null,"config_count":5,"mean":-0.1092,"ci_low":-0.2376,"ci_high":0.0362,"score_points_mean":-2.748,"score_points_ci_low":-5.981,"score_points_ci_high":0.911,"p_gt_0":0.0686},{"condition":"disabled:high","level":"disabled:high","ordinal_rank":null,"config_count":5,"mean":-0.1465,"ci_low":-0.2113,"ci_high":-0.0696,"score_points_mean":-3.687,"score_points_ci_low":-5.318,"score_points_ci_high":-1.753,"p_gt_0":0.0006},{"condition":"disabled","level":"disabled","ordinal_rank":null,"config_count":135,"mean":-0.1587,"ci_low":-0.1876,"ci_high":-0.1313,"score_points_mean":-3.995,"score_points_ci_low":-4.723,"score_points_ci_high":-3.304,"p_gt_0":0.0},{"condition":"effort:minimal","level":"effort:minimal","ordinal_rank":0,"config_count":80,"mean":-0.1702,"ci_low":-0.2183,"ci_high":-0.1188,"score_points_mean":-4.285,"score_points_ci_low":-5.494,"score_points_ci_high":-2.989,"p_gt_0":0.0}],"adjacentContrasts":[{"from_condition":"effort:minimal","to_condition":"effort:low","from":"minimal","to":"low","mean":0.084,"ci_low":0.0323,"ci_high":0.1336,"score_points_mean":2.114,"score_points_ci_low":0.812,"score_points_ci_high":3.363,"p_positive":0.9994},{"from_condition":"effort:low","to_condition":"effort:medium","from":"low","to":"medium","mean":0.0714,"ci_low":0.0444,"ci_high":0.0987,"score_points_mean":1.798,"score_points_ci_low":1.118,"score_points_ci_high":2.485,"p_positive":1.0},{"from_condition":"effort:medium","to_condition":"effort:high","from":"medium","to":"high","mean":0.0352,"ci_low":0.0139,"ci_high":0.0564,"score_points_mean":0.885,"score_points_ci_low":0.351,"score_points_ci_high":1.419,"p_positive":0.9997},{"from_condition":"effort:high","to_condition":"effort:xhigh","from":"high","to":"xhigh","mean":0.01,"ci_low":-0.0082,"ci_high":0.0294,"score_points_mean":0.25,"score_points_ci_low":-0.207,"score_points_ci_high":0.741,"p_positive":0.8533},{"from_condition":"effort:xhigh","to_condition":"effort:max","from":"xhigh","to":"max","mean":0.0038,"ci_low":-0.0184,"ci_high":0.0257,"score_points_mean":0.094,"score_points_ci_low":-0.463,"score_points_ci_high":0.648,"p_positive":0.6353}],"summary":{"ordinal_levels":["minimal","low","medium","high","xhigh","max"],"ordinal_conditions":["effort:minimal","effort:low","effort:medium","effort:high","effort:xhigh","effort:max"],"n_ordinal_levels":6,"min_adjacent_p_positive":0.6353,"mean_adjacent_p_positive":0.8975,"p_all_adjacent_positive":0.4975}}},"benchmarks":[{"slug":"rwe-bench","name":"RWE-bench","family":"rwe-bench","signalToNoise":56.532,"beta":13.19,"sigma":0.233,"communality":0.794,"difficulty":174.2,"nModels":9,"precision":0.7,"precisionMidpoint":0.4,"signalPercentile":100},{"slug":"web-clone-bench","name":"Web Clone Bench","family":"web-clone-bench","signalToNoise":47.577,"beta":11.568,"sigma":0.243,"communality":0.832,"difficulty":170.4,"nModels":9,"precision":0.7,"precisionMidpoint":0.5,"signalPercentile":100},{"slug":"dayjob-data-science","name":"DAYJOB: Data Science","family":"dayjob-data-science","signalToNoise":40.924,"beta":8.919,"sigma":0.218,"communality":0.903,"difficulty":169.9,"nModels":13,"precision":0.7,"precisionMidpoint":0.6,"signalPercentile":100},{"slug":"terminal-bench-science-0-1","name":"Terminal-Bench-Science 0.1","family":"terminal-bench-science-0-1","signalToNoise":38.646,"beta":10.337,"sigma":0.267,"communality":0.922,"difficulty":166.4,"nModels":59,"precision":0.7,"precisionMidpoint":0.7,"signalPercentile":100},{"slug":"arxivlean-06-2026-matharena","name":"ArXivLean 06/2026","family":"arxivlean-06-2026-matharena","signalToNoise":38.498,"beta":5.642,"sigma":0.147,"communality":0.606,"difficulty":167.0,"nModels":8,"precision":0.7,"precisionMidpoint":0.7,"signalPercentile":99},{"slug":"terminal-bench-science-vals-retry","name":"Terminal-Bench Science 0.1 (Vals)","family":"terminal-bench-science-vals-retry","signalToNoise":37.643,"beta":8.635,"sigma":0.229,"communality":0.571,"difficulty":168.6,"nModels":41,"precision":0.7,"precisionMidpoint":0.7,"signalPercentile":99},{"slug":"atlas-finance-1-1","name":"ATLAS-Finance v1.1","family":"atlas-finance-1-1","signalToNoise":37.4,"beta":4.692,"sigma":0.125,"communality":0.89,"difficulty":178.7,"nModels":10,"precision":1.1,"precisionMidpoint":0.7,"signalPercentile":99},{"slug":"frontierswe-dart-style-haskell","name":"FrontierSWE (Dominance)","family":"frontierswe","signalToNoise":35.784,"beta":9.516,"sigma":0.266,"communality":0.894,"difficulty":162.8,"nModels":15,"precision":4.8,"precisionMidpoint":0.7,"signalPercentile":99},{"slug":"frontiermath-erdos","name":"FrontierMath Erd\u0151s","family":"frontiermath-erdos","signalToNoise":34.917,"beta":5.238,"sigma":0.15,"communality":0.323,"difficulty":190.0,"nModels":14,"precision":3.3,"precisionMidpoint":0.7,"signalPercentile":99},{"slug":"programbench","name":"ProgramBench","family":"programbench","signalToNoise":33.729,"beta":4.909,"sigma":0.146,"communality":0.411,"difficulty":184.8,"nModels":58,"precision":2.0,"precisionMidpoint":0.7,"signalPercentile":99},{"slug":"frontierphysics-0-1","name":"FrontierPhysics v0.1","family":"frontierphysics-0-1","signalToNoise":33.409,"beta":6.699,"sigma":0.201,"communality":0.456,"difficulty":174.5,"nModels":18,"precision":1.0,"precisionMidpoint":0.8,"signalPercentile":99},{"slug":"srebench","name":"SRE Bench \u2014 Fully Solved","family":"srebench","signalToNoise":32.312,"beta":9.824,"sigma":0.304,"communality":0.669,"difficulty":169.2,"nModels":26,"precision":0.8,"precisionMidpoint":0.8,"signalPercentile":99},{"slug":"gdpx","name":"GDPx","family":"gdpx","signalToNoise":32.253,"beta":3.39,"sigma":0.105,"communality":0.866,"difficulty":185.1,"nModels":17,"precision":1.4,"precisionMidpoint":0.8,"signalPercentile":98},{"slug":"snorkel-swe-bench-cli","name":"Snorkel SWE-bench CLI","family":"snorkel-swe-bench-cli","signalToNoise":31.732,"beta":5.442,"sigma":0.171,"communality":0.787,"difficulty":174.9,"nModels":13,"precision":1.0,"precisionMidpoint":0.8,"signalPercentile":98},{"slug":"frontierswe-v2","name":"FrontierSWE V2","family":"frontierswe-v2","signalToNoise":30.185,"beta":6.118,"sigma":0.203,"communality":0.958,"difficulty":165.7,"nModels":19,"precision":0.8,"precisionMidpoint":0.8,"signalPercentile":98},{"slug":"genebench-pro","name":"GeneBench-Pro","family":"genebench-pro","signalToNoise":29.699,"beta":6.032,"sigma":0.203,"communality":0.897,"difficulty":167.8,"nModels":27,"precision":0.8,"precisionMidpoint":0.8,"signalPercentile":98},{"slug":"decision-bench-1-1","name":"Decision-Bench v1.1","family":"decision-bench-1-1","signalToNoise":29.277,"beta":4.673,"sigma":0.16,"communality":0.885,"difficulty":167.2,"nModels":16,"precision":0.9,"precisionMidpoint":0.9,"signalPercentile":98},{"slug":"txbench-pp-benchmarks-bio","name":"TxBench-PP","family":"txbench-pp-benchmarks-bio","signalToNoise":28.435,"beta":3.957,"sigma":0.139,"communality":0.971,"difficulty":157.7,"nModels":11,"precision":1.1,"precisionMidpoint":0.9,"signalPercentile":98},{"slug":"dayjob-finance","name":"DAYJOB: Finance","family":"dayjob-finance","signalToNoise":28.31,"beta":6.77,"sigma":0.239,"communality":0.735,"difficulty":171.5,"nModels":31,"precision":1.0,"precisionMidpoint":0.9,"signalPercentile":98},{"slug":"matflowbench","name":"MatFlowBench Pass@1","family":"matflowbench","signalToNoise":27.571,"beta":5.924,"sigma":0.215,"communality":0.94,"difficulty":166.2,"nModels":9,"precision":0.9,"precisionMidpoint":0.9,"signalPercentile":97},{"slug":"remote-labor-index","name":"Remote Labor Index (RLI)","family":"remote-labor-index","signalToNoise":27.204,"beta":3.903,"sigma":0.143,"communality":0.903,"difficulty":175.9,"nModels":16,"precision":1.1,"precisionMidpoint":0.9,"signalPercentile":97},{"slug":"dayjob-healthcare","name":"DAYJOB: Healthcare","family":"dayjob-healthcare","signalToNoise":26.256,"beta":7.079,"sigma":0.27,"communality":0.587,"difficulty":173.5,"nModels":31,"precision":1.4,"precisionMidpoint":1.0,"signalPercentile":97},{"slug":"enigmaeval-scale-ai-2","name":"EnigmaEval","family":"enigmaeval-scale-ai-2","signalToNoise":24.379,"beta":3.731,"sigma":0.153,"communality":0.963,"difficulty":166.6,"nModels":48,"precision":1.1,"precisionMidpoint":1.0,"signalPercentile":97},{"slug":"bankertoolbench-1-1","name":"BankerToolBench v1.1","family":"bankertoolbench-1-1","signalToNoise":24.239,"beta":4.254,"sigma":0.176,"communality":0.93,"difficulty":168.5,"nModels":11,"precision":1.0,"precisionMidpoint":1.0,"signalPercentile":97},{"slug":"design-creativity-bench-appropriateness","name":"Design Creativity Bench","family":"design-creativity-bench","signalToNoise":23.172,"beta":2.443,"sigma":0.105,"communality":0.833,"difficulty":123.4,"nModels":14,"precision":5.0,"precisionMidpoint":1.1,"signalPercentile":97},{"slug":"medqa-4options-openmedical-harness","name":"MedQA 4-options (Open Medical-LLM)","family":"medqa-4options-openmedical-harness","signalToNoise":22.916,"beta":2.34,"sigma":0.102,"communality":0.974,"difficulty":105.2,"nModels":17,"precision":12.0,"precisionMidpoint":1.1,"signalPercentile":97},{"slug":"pg-llm-50","name":"PG-LLM-50","family":"pg-llm-50","signalToNoise":22.471,"beta":1.458,"sigma":0.065,"communality":0.839,"difficulty":172.6,"nModels":16,"precision":2.0,"precisionMidpoint":1.1,"signalPercentile":97},{"slug":"rsi-exam-0-1","name":"RSI-Exam 0.1 (Full 88)","family":"rsi-exam-0-1","signalToNoise":21.156,"beta":2.335,"sigma":0.11,"communality":0.899,"difficulty":166.9,"nModels":17,"precision":1.2,"precisionMidpoint":1.2,"signalPercentile":96},{"slug":"android-bench-2","name":"Android Bench 2.0","family":"android-bench-2","signalToNoise":20.875,"beta":4.43,"sigma":0.212,"communality":0.916,"difficulty":172.4,"nModels":9,"precision":1.3,"precisionMidpoint":1.2,"signalPercentile":96},{"slug":"terminal-bench-3-reported","name":"Terminal-Bench 3 - release-reported accuracy","family":"terminal-bench-3-reported","signalToNoise":20.834,"beta":5.134,"sigma":0.246,"communality":0.813,"difficulty":167.6,"nModels":9,"precision":1.2,"precisionMidpoint":1.2,"signalPercentile":96},{"slug":"cursorbench-4-0","name":"CursorBench 4.0","family":"cursorbench-4-0","signalToNoise":20.72,"beta":2.667,"sigma":0.129,"communality":0.904,"difficulty":165.4,"nModels":19,"precision":1.2,"precisionMidpoint":1.2,"signalPercentile":96},{"slug":"big-bench-extra-hard","name":"BIG-Bench Extra Hard","family":"big-bench-extra-hard","signalToNoise":20.705,"beta":2.923,"sigma":0.141,"communality":0.98,"difficulty":140.6,"nModels":15,"precision":3.2,"precisionMidpoint":1.2,"signalPercentile":96},{"slug":"weirdml-v3","name":"WeirdML v3","family":"weirdml-v3","signalToNoise":20.163,"beta":4.494,"sigma":0.223,"communality":0.865,"difficulty":173.3,"nModels":24,"precision":1.3,"precisionMidpoint":1.2,"signalPercentile":96},{"slug":"osworld-2-1-full-reported-partial","name":"OSWorld 2.1 Full, 500-step Binary Accuracy","family":"osworld-2-1-full-500-binary","signalToNoise":19.772,"beta":2.875,"sigma":0.145,"communality":0.864,"difficulty":153.9,"nModels":8,"precision":1.7,"precisionMidpoint":1.3,"signalPercentile":96},{"slug":"sec-bench-pro-reported","name":"SEC-Bench Pro (CAISI 183 tasks)","family":"sec-bench-pro-caisi-183","signalToNoise":19.669,"beta":5.119,"sigma":0.26,"communality":0.874,"difficulty":157.8,"nModels":10,"precision":1.9,"precisionMidpoint":1.3,"signalPercentile":95},{"slug":"pencil-puzzle-bench-agentic-ppbench","name":"Pencil Puzzle Bench - Agentic","family":"pencil-puzzle-bench-agentic-ppbench","signalToNoise":19.613,"beta":8.208,"sigma":0.419,"communality":0.533,"difficulty":157.6,"nModels":49,"precision":3.4,"precisionMidpoint":1.3,"signalPercentile":95},{"slug":"atlas-finance","name":"ATLAS-Finance v1.0","family":"atlas-finance","signalToNoise":19.496,"beta":4.203,"sigma":0.216,"communality":0.611,"difficulty":180.3,"nModels":10,"precision":2.0,"precisionMidpoint":1.3,"signalPercentile":95},{"slug":"time-horizon-index-ksp","name":"Time Horizon Index: KSP","family":"time-horizon-index-ksp","signalToNoise":19.257,"beta":9.799,"sigma":0.509,"communality":0.836,"difficulty":166.3,"nModels":13,"precision":1.4,"precisionMidpoint":1.3,"signalPercentile":95},{"slug":"terminal-bench-4-0","name":"Terminal-Bench 4.0","family":"terminal-bench-4-0","signalToNoise":19.2,"beta":6.729,"sigma":0.35,"communality":0.804,"difficulty":165.2,"nModels":63,"precision":1.4,"precisionMidpoint":1.3,"signalPercentile":95},{"slug":"frontier-bench-0-1","name":"Frontier-Bench v0.1","family":"frontier-bench-0-1","signalToNoise":19.195,"beta":4.495,"sigma":0.234,"communality":0.792,"difficulty":170.5,"nModels":8,"precision":1.3,"precisionMidpoint":1.3,"signalPercentile":95},{"slug":"ple","name":"PLE (Physical-Engineering's Last Exam)","family":"ple","signalToNoise":18.775,"beta":2.985,"sigma":0.159,"communality":0.743,"difficulty":168.7,"nModels":14,"precision":1.3,"precisionMidpoint":1.3,"signalPercentile":95},{"slug":"pumac-2024-caisi-55","name":"PUMaC 2024 (CAISI 55-task set)","family":"pumac-2024-caisi-55","signalToNoise":18.431,"beta":3.259,"sigma":0.177,"communality":0.947,"difficulty":134.1,"nModels":13,"precision":6.3,"precisionMidpoint":1.4,"signalPercentile":95},{"slug":"dayjob-tax","name":"DAYJOB: Tax","family":"dayjob-tax","signalToNoise":18.027,"beta":4.536,"sigma":0.252,"communality":0.354,"difficulty":184.2,"nModels":14,"precision":3.2,"precisionMidpoint":1.4,"signalPercentile":94},{"slug":"scbench-long-benchmarks-bio","name":"ScBench-Long","family":"scbench-long-benchmarks-bio","signalToNoise":17.856,"beta":4.544,"sigma":0.254,"communality":0.856,"difficulty":168.0,"nModels":19,"precision":1.4,"precisionMidpoint":1.4,"signalPercentile":94},{"slug":"vpct","name":"VPCT","family":"vpct","signalToNoise":17.785,"beta":5.791,"sigma":0.326,"communality":0.759,"difficulty":153.3,"nModels":25,"precision":5.6,"precisionMidpoint":1.4,"signalPercentile":94},{"slug":"terminal-bench-4-0-artificial-analysis","name":"Terminal-Bench 4.0 (Artificial Analysis)","family":"terminal-bench-4-0-artificial-analysis","signalToNoise":17.75,"beta":6.684,"sigma":0.377,"communality":0.803,"difficulty":165.8,"nModels":146,"precision":1.5,"precisionMidpoint":1.4,"signalPercentile":94},{"slug":"frontiercode-2-0-score-cognition","name":"FrontierCode 2.0 - Score","family":"frontiercode-2-0-score-cognition","signalToNoise":17.36,"beta":3.917,"sigma":0.226,"communality":0.56,"difficulty":170.7,"nModels":22,"precision":1.5,"precisionMidpoint":1.5,"signalPercentile":94},{"slug":"gsm1k","name":"GSM1k","family":"gsm1k","signalToNoise":17.068,"beta":2.52,"sigma":0.148,"communality":0.938,"difficulty":96.4,"nModels":16,"precision":27.1,"precisionMidpoint":1.5,"signalPercentile":94},{"slug":"otis-aime-2025-caisi-30","name":"OTIS-AIME 2025 (CAISI 30-task set)","family":"otis-aime-2025-caisi-30","signalToNoise":16.86,"beta":3.799,"sigma":0.225,"communality":0.622,"difficulty":135.4,"nModels":13,"precision":9.1,"precisionMidpoint":1.5,"signalPercentile":94},{"slug":"deepswe-datacurve-2","name":"DeepSWE v1","family":"deepswe-datacurve-2","signalToNoise":16.66,"beta":6.851,"sigma":0.411,"communality":0.787,"difficulty":158.9,"nModels":21,"precision":2.6,"precisionMidpoint":1.5,"signalPercentile":93},{"slug":"handbook-md-surge-ai-2","name":"HANDBOOK.md","family":"handbook-md-surge-ai-2","signalToNoise":16.371,"beta":3.933,"sigma":0.24,"communality":0.833,"difficulty":170.4,"nModels":30,"precision":1.6,"precisionMidpoint":1.5,"signalPercentile":93},{"slug":"revelo-code-index","name":"Revelo Code Index","family":"revelo-code-index","signalToNoise":16.074,"beta":3.194,"sigma":0.199,"communality":0.851,"difficulty":165.9,"nModels":19,"precision":1.6,"precisionMidpoint":1.6,"signalPercentile":93},{"slug":"apex-matharena-2","name":"Apex (MathArena)","family":"apex-matharena-2","signalToNoise":15.987,"beta":6.257,"sigma":0.391,"communality":0.676,"difficulty":159.3,"nModels":49,"precision":2.8,"precisionMidpoint":1.6,"signalPercentile":93},{"slug":"swe-sweep","name":"SWE-sweep","family":"swe-sweep","signalToNoise":15.888,"beta":2.814,"sigma":0.177,"communality":0.811,"difficulty":196.9,"nModels":11,"precision":3.7,"precisionMidpoint":1.6,"signalPercentile":93},{"slug":"cursorbench-3-2","name":"CursorBench 3.2","family":"cursorbench-3-2","signalToNoise":15.673,"beta":2.044,"sigma":0.13,"communality":0.862,"difficulty":154.9,"nModels":17,"precision":1.8,"precisionMidpoint":1.6,"signalPercentile":93},{"slug":"mmlu-college-biology-openmedical-harness","name":"MMLU College Biology (Open Medical-LLM)","family":"mmlu-college-biology-openmedical-harness","signalToNoise":15.661,"beta":2.622,"sigma":0.167,"communality":0.966,"difficulty":94.2,"nModels":17,"precision":44.2,"precisionMidpoint":1.6,"signalPercentile":93},{"slug":"mmlu-college-medicine-openmedical-harness","name":"MMLU College Medicine (Open Medical-LLM)","family":"mmlu-college-medicine-openmedical-harness","signalToNoise":15.489,"beta":2.263,"sigma":0.146,"communality":0.967,"difficulty":99.9,"nModels":17,"precision":20.4,"precisionMidpoint":1.6,"signalPercentile":93},{"slug":"critpt","name":"CritPt (avg@5)","family":"critpt","signalToNoise":15.298,"beta":3.748,"sigma":0.245,"communality":0.702,"difficulty":171.1,"nModels":388,"precision":1.7,"precisionMidpoint":1.6,"signalPercentile":92},{"slug":"intphys-2","name":"IntPhys 2","family":"intphys-2","signalToNoise":15.289,"beta":2.439,"sigma":0.16,"communality":0.702,"difficulty":171.4,"nModels":51,"precision":3.0,"precisionMidpoint":1.6,"signalPercentile":92},{"slug":"frontierml","name":"FrontierML ($200 wallet)","family":"frontierml","signalToNoise":15.098,"beta":5.337,"sigma":0.354,"communality":0.761,"difficulty":168.4,"nModels":18,"precision":1.7,"precisionMidpoint":1.7,"signalPercentile":92},{"slug":"frontiermath-tier-4-v1-private","name":"FrontierMath Tier 4 (v1) - Private","family":"frontiermath-tier-4-v1-private","signalToNoise":14.919,"beta":2.856,"sigma":0.191,"communality":0.879,"difficulty":167.8,"nModels":9,"precision":1.7,"precisionMidpoint":1.7,"signalPercentile":92},{"slug":"frontiercode-extended-score-cognition-2","name":"FrontierCode 1.0 Extended - Score","family":"frontiercode-extended-score-cognition-2","signalToNoise":14.737,"beta":2.511,"sigma":0.17,"communality":0.931,"difficulty":160.8,"nModels":15,"precision":1.8,"precisionMidpoint":1.7,"signalPercentile":92},{"slug":"bizagentbench","name":"BizAgentBench (strict success)","family":"bizagentbench","signalToNoise":14.728,"beta":6.173,"sigma":0.419,"communality":0.746,"difficulty":161.9,"nModels":8,"precision":2.3,"precisionMidpoint":1.7,"signalPercentile":92},{"slug":"vibench-2-0","name":"ViBench 2.0","family":"vibench-2-0","signalToNoise":14.612,"beta":8.848,"sigma":0.606,"communality":0.256,"difficulty":168.7,"nModels":9,"precision":1.7,"precisionMidpoint":1.7,"signalPercentile":92},{"slug":"hackerrank-bench","name":"HackerRank-Bench","family":"hackerrank-bench","signalToNoise":14.585,"beta":2.505,"sigma":0.172,"communality":0.869,"difficulty":168.2,"nModels":15,"precision":1.7,"precisionMidpoint":1.7,"signalPercentile":91},{"slug":"mmlu-medical-genetics-openmedical-harness","name":"MMLU Medical Genetics (Open Medical-LLM)","family":"mmlu-medical-genetics-openmedical-harness","signalToNoise":14.576,"beta":2.509,"sigma":0.172,"communality":0.964,"difficulty":93.1,"nModels":17,"precision":42.4,"precisionMidpoint":1.7,"signalPercentile":91},{"slug":"code-migration-v1-1","name":"Code Migration v1.1","family":"code-migration-v1-1","signalToNoise":14.518,"beta":4.822,"sigma":0.332,"communality":0.87,"difficulty":160.7,"nModels":27,"precision":2.1,"precisionMidpoint":1.7,"signalPercentile":91},{"slug":"vals-rsi-index-v1-1-four-tasks","name":"Vals RSI Index v1.1 (four tasks)","family":"vals-rsi-index-v1-1-four-tasks","signalToNoise":14.508,"beta":2.257,"sigma":0.156,"communality":0.842,"difficulty":175.7,"nModels":23,"precision":1.9,"precisionMidpoint":1.7,"signalPercentile":91},{"slug":"missionreport","name":"MissionReport","family":"missionreport","signalToNoise":14.489,"beta":1.194,"sigma":0.082,"communality":0.859,"difficulty":153.4,"nModels":9,"precision":1.8,"precisionMidpoint":1.7,"signalPercentile":91},{"slug":"osworld-2-500-step-partial-score","name":"OSWorld 2.0","family":"osworld-2","signalToNoise":14.428,"beta":4.262,"sigma":0.295,"communality":0.905,"difficulty":161.1,"nModels":13,"precision":2.1,"precisionMidpoint":1.7,"signalPercentile":91},{"slug":"arc-agi-2","name":"ARC-AGI-2","family":"arc-agi-2","signalToNoise":14.162,"beta":5.685,"sigma":0.401,"communality":0.863,"difficulty":157.2,"nModels":93,"precision":3.8,"precisionMidpoint":1.8,"signalPercentile":91},{"slug":"sudo-l7-surge-ai","name":"sudo L7","family":"sudo-l7-surge-ai","signalToNoise":14.143,"beta":3.691,"sigma":0.261,"communality":0.787,"difficulty":173.6,"nModels":13,"precision":2.0,"precisionMidpoint":1.8,"signalPercentile":91},{"slug":"gso","name":"GSO Opt@1","family":"gso","signalToNoise":14.123,"beta":4.503,"sigma":0.319,"communality":0.898,"difficulty":159.6,"nModels":29,"precision":2.2,"precisionMidpoint":1.8,"signalPercentile":90},{"slug":"mcpmark","name":"MCPMark","family":"mcpmark","signalToNoise":14.07,"beta":3.61,"sigma":0.257,"communality":0.821,"difficulty":152.7,"nModels":22,"precision":3.0,"precisionMidpoint":1.8,"signalPercentile":90},{"slug":"cl-bench-life","name":"CL-bench Life (Overall)","family":"cl-bench-life","signalToNoise":13.964,"beta":1.842,"sigma":0.132,"communality":0.78,"difficulty":177.4,"nModels":15,"precision":1.9,"precisionMidpoint":1.8,"signalPercentile":90},{"slug":"babaisbench-lake","name":"BabaIsBench Stage 1: The Lake","family":"babaisbench-lake","signalToNoise":13.892,"beta":6.913,"sigma":0.498,"communality":0.42,"difficulty":154.8,"nModels":19,"precision":5.5,"precisionMidpoint":1.8,"signalPercentile":90},{"slug":"arc-agi-1-semi-private","name":"ARC-AGI-1 Semi-Private","family":"arc-agi-1-semi-private","signalToNoise":13.561,"beta":4.44,"sigma":0.327,"communality":0.932,"difficulty":147.6,"nModels":89,"precision":6.8,"precisionMidpoint":1.9,"signalPercentile":90},{"slug":"usamo-2026","name":"USAMO 2026","family":"usamo-2026","signalToNoise":13.492,"beta":6.982,"sigma":0.518,"communality":0.706,"difficulty":151.5,"nModels":13,"precision":9.3,"precisionMidpoint":1.9,"signalPercentile":90},{"slug":"oncologybench-strict-25-oct9","name":"OncologyBench Strict (25 tasks, October 9 rubric)","family":"oncologybench-strict-25-oct9","signalToNoise":13.482,"beta":3.295,"sigma":0.244,"communality":0.686,"difficulty":173.2,"nModels":26,"precision":2.0,"precisionMidpoint":1.9,"signalPercentile":90},{"slug":"agents-last-exam-ale","name":"Agents' Last Exam (ALE)","family":"agents-last-exam-ale","signalToNoise":13.413,"beta":2.393,"sigma":0.178,"communality":0.834,"difficulty":172.2,"nModels":30,"precision":1.9,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"nl2repo-reported","name":"NL2Repo - release-reported repository generation score","family":"nl2repo-reported","signalToNoise":13.381,"beta":2.637,"sigma":0.197,"communality":0.892,"difficulty":156.9,"nModels":29,"precision":2.2,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"reactbench-39","name":"ReactBench (39 tasks)","family":"reactbench-39","signalToNoise":13.364,"beta":2.442,"sigma":0.183,"communality":0.793,"difficulty":168.2,"nModels":27,"precision":1.9,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"toolathlon-verified","name":"Toolathlon-Verified","family":"toolathlon-verified","signalToNoise":13.335,"beta":2.894,"sigma":0.217,"communality":0.879,"difficulty":151.1,"nModels":38,"precision":2.8,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"zerobench-main-mean-pass-1-five-samples","name":"ZeroBench (Main questions, pass@1)","family":"zerobench-main-questions-pass-1-zerobench","signalToNoise":13.33,"beta":2.715,"sigma":0.204,"communality":0.808,"difficulty":177.6,"nModels":49,"precision":2.1,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"quality","name":"QuALITY","family":"quality","signalToNoise":13.3,"beta":1.378,"sigma":0.104,"communality":0.963,"difficulty":91.1,"nModels":8,"precision":9.2,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"hedge-bench","name":"Hedge-Bench","family":"hedge-bench","signalToNoise":13.22,"beta":5.058,"sigma":0.383,"communality":0.653,"difficulty":171.1,"nModels":12,"precision":2.0,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"astabench-2026-core-hard","name":"CORE-Bench-Hard (AstaBench 2026)","family":"astabench-2026-core-hard","signalToNoise":13.159,"beta":3.933,"sigma":0.299,"communality":0.735,"difficulty":146.7,"nModels":13,"precision":5.7,"precisionMidpoint":1.9,"signalPercentile":89},{"slug":"swe-atlas-codebase-qna-mercor-extended","name":"SWE Atlas Codebase QnA Extended (Mercor held-out)","family":"swe-atlas-codebase-qna-mercor-extended","signalToNoise":13.127,"beta":4.4,"sigma":0.335,"communality":0.459,"difficulty":168.8,"nModels":33,"precision":1.9,"precisionMidpoint":1.9,"signalPercentile":88},{"slug":"global-piqa-parallel-loglikelihood-accuracy","name":"Global PIQA - Parallel (Log-Likelihood Accuracy)","family":"global-piqa-parallel-loglikelihood-accuracy","signalToNoise":13.118,"beta":1.68,"sigma":0.128,"communality":0.401,"difficulty":166.5,"nModels":20,"precision":2.5,"precisionMidpoint":1.9,"signalPercentile":88},{"slug":"diligence-stack-agent","name":"Diligence Stack Agent Bench","family":"diligence-stack-agent","signalToNoise":13.067,"beta":3.118,"sigma":0.239,"communality":0.816,"difficulty":149.2,"nModels":14,"precision":3.5,"precisionMidpoint":1.9,"signalPercentile":88},{"slug":"aa-3d-v1","name":"AA-3D v1.0 Elo","family":"aa-3d-v1","signalToNoise":13.062,"beta":4.48,"sigma":0.343,"communality":0.924,"difficulty":161.6,"nModels":23,"precision":1.9,"precisionMidpoint":1.9,"signalPercentile":88},{"slug":"gpqa-openllm-harness","name":"GPQA (Open LLM Leaderboard v2)","family":"gpqa-openllm-harness","signalToNoise":13.004,"beta":1.336,"sigma":0.103,"communality":0.719,"difficulty":153.5,"nModels":121,"precision":2.5,"precisionMidpoint":1.9,"signalPercentile":88},{"slug":"mmlu-clinical-knowledge-openmedical-harness","name":"MMLU Clinical Knowledge (Open Medical-LLM)","family":"mmlu-clinical-knowledge-openmedical-harness","signalToNoise":12.935,"beta":2.251,"sigma":0.174,"communality":0.964,"difficulty":94.8,"nModels":17,"precision":30.2,"precisionMidpoint":1.9,"signalPercentile":88},{"slug":"code2video-aa","name":"Code2Video-AA Elo","family":"code2video-aa","signalToNoise":12.923,"beta":4.707,"sigma":0.364,"communality":0.903,"difficulty":160.7,"nModels":27,"precision":1.9,"precisionMidpoint":1.9,"signalPercentile":88},{"slug":"frontiercs-2","name":"FrontierCS 2","family":"frontiercs-2","signalToNoise":12.883,"beta":4.161,"sigma":0.323,"communality":0.788,"difficulty":170.6,"nModels":10,"precision":2.0,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"vibe-code","name":"Vibe Code Bench v1.1","family":"vibe-code","signalToNoise":12.785,"beta":5.609,"sigma":0.439,"communality":0.724,"difficulty":155.1,"nModels":109,"precision":4.0,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"supergpqa","name":"SuperGPQA","family":"supergpqa","signalToNoise":12.655,"beta":1.394,"sigma":0.11,"communality":0.975,"difficulty":138.5,"nModels":80,"precision":2.9,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"lmarena-image-to-webdev-arena","name":"Arena Image-to-WebDev","family":"lmarena-image-to-webdev-arena","signalToNoise":12.622,"beta":4.153,"sigma":0.329,"communality":0.91,"difficulty":157.0,"nModels":48,"precision":2.0,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"vcb-1-100","name":"Vibe Code Bench 1\u2013100","family":"vcb-1-100","signalToNoise":12.612,"beta":2.43,"sigma":0.193,"communality":0.762,"difficulty":176.5,"nModels":23,"precision":2.2,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"exploitgym","name":"ExploitGym","family":"exploitgym","signalToNoise":12.585,"beta":4.634,"sigma":0.368,"communality":0.635,"difficulty":172.6,"nModels":10,"precision":2.1,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"proofbench-v1-1","name":"ProofBench v1.1","family":"proofbench-v1-1","signalToNoise":12.49,"beta":7.298,"sigma":0.584,"communality":0.613,"difficulty":159.1,"nModels":50,"precision":4.3,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"btf3-2386-pooled","name":"Bench to the Future 3 \u2014 September Panel \u2014 Pooled","family":"btf3-2386-pooled","signalToNoise":12.461,"beta":1.037,"sigma":0.083,"communality":0.267,"difficulty":112.0,"nModels":8,"precision":3.5,"precisionMidpoint":2.0,"signalPercentile":87},{"slug":"opus-magnum-campaign-opus-magnum-bench-2","name":"Opus Magnum Campaign","family":"opus-magnum-campaign-opus-magnum-bench-2","signalToNoise":12.409,"beta":5.067,"sigma":0.408,"communality":0.46,"difficulty":163.2,"nModels":20,"precision":2.3,"precisionMidpoint":2.0,"signalPercentile":86},{"slug":"spatialviz-bench","name":"SpatialViz-Bench","family":"spatialviz-bench","signalToNoise":12.382,"beta":3.091,"sigma":0.25,"communality":0.888,"difficulty":155.0,"nModels":50,"precision":3.3,"precisionMidpoint":2.0,"signalPercentile":86},{"slug":"ramp-swe-bench-observed-78","name":"Ramp SWE-Bench (78 published tasks)","family":"ramp-swe-bench-observed-78","signalToNoise":12.338,"beta":2.374,"sigma":0.192,"communality":0.848,"difficulty":143.6,"nModels":25,"precision":3.5,"precisionMidpoint":2.0,"signalPercentile":86},{"slug":"taxcalcbench-ty25-web-search","name":"TaxCalcBench TY25 (web search)","family":"taxcalcbench-ty25-web-search","signalToNoise":12.257,"beta":4.055,"sigma":0.331,"communality":0.856,"difficulty":166.6,"nModels":20,"precision":2.2,"precisionMidpoint":2.1,"signalPercentile":86},{"slug":"biosecbench-function-v2","name":"BioSecBench-Function V1 Clean Reruns","family":"biosecbench-function-v2","signalToNoise":12.114,"beta":2.866,"sigma":0.237,"communality":0.727,"difficulty":165.7,"nModels":24,"precision":2.1,"precisionMidpoint":2.1,"signalPercentile":86},{"slug":"snorkel-agentic-coding-2","name":"Snorkel Agentic Coding 2.0","family":"snorkel-agentic-coding-2","signalToNoise":12.091,"beta":2.429,"sigma":0.201,"communality":0.676,"difficulty":170.0,"nModels":13,"precision":2.1,"precisionMidpoint":2.1,"signalPercentile":86},{"slug":"ioi-vals-v2","name":"IOI (Vals v2)","family":"ioi-vals-v2","signalToNoise":12.023,"beta":5.205,"sigma":0.433,"communality":0.51,"difficulty":159.8,"nModels":43,"precision":3.2,"precisionMidpoint":2.1,"signalPercentile":86},{"slug":"gdp-pdf","name":"GDP.pdf","family":"gdp-pdf","signalToNoise":12.018,"beta":2.066,"sigma":0.172,"communality":0.899,"difficulty":176.1,"nModels":39,"precision":2.2,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"frontiermath-tiers-1-3-v1-private","name":"FrontierMath Tiers 1-3 (v1) - Private","family":"frontiermath-tiers-1-3-v1-private","signalToNoise":12.016,"beta":1.648,"sigma":0.137,"communality":0.916,"difficulty":162.0,"nModels":8,"precision":2.1,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"vulcanbench-frontier-v4","name":"VulcanBench Frontier v4","family":"vulcanbench-frontier-v4","signalToNoise":11.997,"beta":3.862,"sigma":0.322,"communality":0.704,"difficulty":152.2,"nModels":12,"precision":3.8,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"autoresearchexam","name":"AutoResearchExam (hidden-test AUARC)","family":"autoresearchexam","signalToNoise":11.983,"beta":1.984,"sigma":0.166,"communality":0.799,"difficulty":162.3,"nModels":11,"precision":2.2,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"charxiv-reasoning-with-tools-card","name":"CharXiv Reasoning (with tools)","family":"charxiv-reasoning-with-tools-card","signalToNoise":11.963,"beta":1.859,"sigma":0.155,"communality":0.834,"difficulty":131.3,"nModels":41,"precision":4.4,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"dayjob-accounting","name":"DAYJOB: Accounting","family":"dayjob-accounting","signalToNoise":11.903,"beta":3.966,"sigma":0.333,"communality":0.656,"difficulty":174.8,"nModels":13,"precision":2.4,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"agieval-en","name":"AGIEval English","family":"agieval-en","signalToNoise":11.835,"beta":1.592,"sigma":0.135,"communality":0.975,"difficulty":113.9,"nModels":36,"precision":7.0,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"mmlu-professional-medicine-openmedical-harness","name":"MMLU Professional Medicine (Open Medical-LLM)","family":"mmlu-professional-medicine-openmedical-harness","signalToNoise":11.828,"beta":3.028,"sigma":0.256,"communality":0.927,"difficulty":97.2,"nModels":17,"precision":88.7,"precisionMidpoint":2.1,"signalPercentile":85},{"slug":"claw-eval-mm-pass-3","name":"ClawEval-MM - pass@3","family":"claw-eval-mm-pass-3","signalToNoise":11.819,"beta":2.916,"sigma":0.247,"communality":0.817,"difficulty":152.2,"nModels":9,"precision":3.1,"precisionMidpoint":2.1,"signalPercentile":84},{"slug":"ioi","name":"IOI","family":"ioi","signalToNoise":11.801,"beta":4.674,"sigma":0.396,"communality":0.705,"difficulty":156.5,"nModels":53,"precision":3.3,"precisionMidpoint":2.1,"signalPercentile":84},{"slug":"medmcqa-openmedical-harness","name":"MedMCQA (Open Medical-LLM)","family":"medmcqa-openmedical-harness","signalToNoise":11.79,"beta":1.661,"sigma":0.141,"communality":0.907,"difficulty":110.5,"nModels":17,"precision":8.5,"precisionMidpoint":2.1,"signalPercentile":84},{"slug":"visualtoolbench-vtb-scale-ai-2","name":"VisualToolBench (VTB)","family":"visualtoolbench-vtb-scale-ai-2","signalToNoise":11.784,"beta":2.985,"sigma":0.253,"communality":0.862,"difficulty":164.5,"nModels":17,"precision":2.2,"precisionMidpoint":2.1,"signalPercentile":84},{"slug":"mmmu-pro-mercor-extended","name":"MMMU-Pro Extended (Mercor held-out)","family":"mmmu-pro-mercor-extended","signalToNoise":11.73,"beta":1.948,"sigma":0.166,"communality":0.89,"difficulty":170.4,"nModels":36,"precision":2.1,"precisionMidpoint":2.1,"signalPercentile":84},{"slug":"ojbench-cpp","name":"OJBench (C++)","family":"ojbench-cpp","signalToNoise":11.706,"beta":2.536,"sigma":0.217,"communality":0.897,"difficulty":149.8,"nModels":9,"precision":3.4,"precisionMidpoint":2.2,"signalPercentile":84},{"slug":"cua-bench","name":"CUA-bench","family":"cua-bench","signalToNoise":11.681,"beta":2.842,"sigma":0.243,"communality":0.539,"difficulty":189.1,"nModels":9,"precision":3.8,"precisionMidpoint":2.2,"signalPercentile":84},{"slug":"aurora-multilingual-terminal-bench","name":"AURORA Multilingual Terminal-Bench","family":"aurora-multilingual-terminal-bench","signalToNoise":11.615,"beta":2.59,"sigma":0.223,"communality":0.918,"difficulty":155.6,"nModels":10,"precision":2.7,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"gdp-pdf-all-pass-artificial-analysis","name":"GDP.pdf (Artificial Analysis, All-pass)","family":"gdp-pdf-all-pass-artificial-analysis","signalToNoise":11.559,"beta":2.392,"sigma":0.207,"communality":0.776,"difficulty":175.8,"nModels":146,"precision":2.3,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"proof_bench","name":"ProofBench","family":"proof_bench","signalToNoise":11.52,"beta":5.107,"sigma":0.443,"communality":0.656,"difficulty":159.0,"nModels":51,"precision":3.0,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"critpt-v2","name":"CritPt v2 (agentic, avg@5)","family":"critpt-v2","signalToNoise":11.515,"beta":3.98,"sigma":0.346,"communality":0.872,"difficulty":159.1,"nModels":50,"precision":2.8,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"biomysterybench-vals","name":"BioMysteryBench (Vals Terminus 2)","family":"biomysterybench-vals","signalToNoise":11.509,"beta":2.114,"sigma":0.184,"communality":0.743,"difficulty":151.7,"nModels":27,"precision":2.7,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"hle-mercor-extended","name":"HLE Extended (Mercor held-out)","family":"hle-mercor-extended","signalToNoise":11.504,"beta":5.015,"sigma":0.436,"communality":0.748,"difficulty":162.3,"nModels":28,"precision":2.7,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"hmmt-feb-2026-matharena","name":"HMMT Feb 2026","family":"hmmt-feb-2026-matharena","signalToNoise":11.401,"beta":2.848,"sigma":0.25,"communality":0.835,"difficulty":134.3,"nModels":44,"precision":8.8,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"swe-marathon","name":"SWE-Marathon v1.0","family":"swe-marathon","signalToNoise":11.308,"beta":4.106,"sigma":0.363,"communality":0.305,"difficulty":171.6,"nModels":12,"precision":2.4,"precisionMidpoint":2.2,"signalPercentile":83},{"slug":"aider-code-editing","name":"Aider Code Editing","family":"aider-code-editing","signalToNoise":11.279,"beta":1.705,"sigma":0.151,"communality":0.961,"difficulty":109.2,"nModels":8,"precision":8.4,"precisionMidpoint":2.2,"signalPercentile":82},{"slug":"taxcalcbench-ty25","name":"TaxCalcBench TY25","family":"taxcalcbench-ty25","signalToNoise":11.269,"beta":4.002,"sigma":0.355,"communality":0.41,"difficulty":174.9,"nModels":26,"precision":2.3,"precisionMidpoint":2.2,"signalPercentile":82},{"slug":"swe-atlas-test-writing-scale-ai-2","name":"SWE Atlas - Test Writing","family":"swe-atlas-test-writing-scale-ai-2","signalToNoise":11.147,"beta":2.14,"sigma":0.192,"communality":0.879,"difficulty":162.0,"nModels":23,"precision":2.3,"precisionMidpoint":2.3,"signalPercentile":82},{"slug":"deepswe-1-1","name":"DeepSWE v1.1","family":"deepswe-1-1","signalToNoise":10.979,"beta":5.062,"sigma":0.461,"communality":0.646,"difficulty":159.1,"nModels":72,"precision":3.3,"precisionMidpoint":2.3,"signalPercentile":82},{"slug":"spatialbench-long-benchmarks-bio","name":"SpatialBench-Long","family":"spatialbench-long-benchmarks-bio","signalToNoise":10.96,"beta":2.074,"sigma":0.189,"communality":0.56,"difficulty":190.6,"nModels":12,"precision":3.3,"precisionMidpoint":2.3,"signalPercentile":82},{"slug":"furniture-assembly","name":"Furniture Assembly Benchmark","family":"furniture-assembly","signalToNoise":10.918,"beta":3.879,"sigma":0.355,"communality":0.813,"difficulty":164.4,"nModels":35,"precision":2.5,"precisionMidpoint":2.3,"signalPercentile":82},{"slug":"orca-bench-hard-rca-fireworks","name":"ORCA-bench Hard RCA (Fireworks 324-task subset)","family":"orca-bench-hard-rca-fireworks","signalToNoise":10.897,"beta":4.29,"sigma":0.394,"communality":0.455,"difficulty":168.7,"nModels":12,"precision":2.3,"precisionMidpoint":2.3,"signalPercentile":82},{"slug":"automationbench-1-0-6-zapier","name":"AutomationBench (Overall)","family":"automationbench-overall-zapier-benchmarks","signalToNoise":10.89,"beta":4.243,"sigma":0.39,"communality":0.627,"difficulty":170.4,"nModels":36,"precision":2.4,"precisionMidpoint":2.3,"signalPercentile":81},{"slug":"arxivmath-overall-matharena","name":"ArXivMath \u2014 Overall","family":"arxivmath-overall-matharena","signalToNoise":10.843,"beta":2.44,"sigma":0.225,"communality":0.96,"difficulty":154.4,"nModels":12,"precision":2.9,"precisionMidpoint":2.3,"signalPercentile":81},{"slug":"frontiercode-1-1-extended-score-cognition","name":"FrontierCode 1.1 Extended - Score","family":"frontiercode-1-1-extended-score-cognition","signalToNoise":10.834,"beta":1.936,"sigma":0.179,"communality":0.825,"difficulty":158.7,"nModels":39,"precision":2.5,"precisionMidpoint":2.3,"signalPercentile":81},{"slug":"gdpval","name":"GDPval","family":"gdpval","signalToNoise":10.793,"beta":3.051,"sigma":0.283,"communality":0.769,"difficulty":147.9,"nModels":10,"precision":4.3,"precisionMidpoint":2.3,"signalPercentile":81},{"slug":"biosecbench-surveillance-benchmarks-bio","name":"BioSecBench-Surveillance","family":"biosecbench-surveillance-benchmarks-bio","signalToNoise":10.784,"beta":2.81,"sigma":0.261,"communality":0.77,"difficulty":159.6,"nModels":17,"precision":2.6,"precisionMidpoint":2.3,"signalPercentile":81},{"slug":"mmlu-anatomy-openmedical-harness","name":"MMLU Anatomy (Open Medical-LLM)","family":"mmlu-anatomy-openmedical-harness","signalToNoise":10.72,"beta":1.846,"sigma":0.172,"communality":0.929,"difficulty":102.8,"nModels":17,"precision":15.2,"precisionMidpoint":2.3,"signalPercentile":81},{"slug":"smt-2025-caisi-58","name":"SMT 2025 (CAISI 58-task set)","family":"smt-2025-caisi-58","signalToNoise":10.706,"beta":2.23,"sigma":0.208,"communality":0.759,"difficulty":122.4,"nModels":13,"precision":9.1,"precisionMidpoint":2.4,"signalPercentile":81},{"slug":"slopcodebench-core-solved-scbench","name":"SlopCodeBench - Core Solved","family":"slopcodebench-core-solved-scbench","signalToNoise":10.676,"beta":3.088,"sigma":0.289,"communality":0.812,"difficulty":153.6,"nModels":14,"precision":3.3,"precisionMidpoint":2.4,"signalPercentile":81},{"slug":"variantbench-benchmarks-bio","name":"VariantBench","family":"variantbench-benchmarks-bio","signalToNoise":10.565,"beta":1.741,"sigma":0.165,"communality":0.826,"difficulty":167.9,"nModels":15,"precision":2.4,"precisionMidpoint":2.4,"signalPercentile":80},{"slug":"pencil-puzzle-bench-direct-ask-ppbench","name":"Pencil Puzzle Bench - Direct Ask","family":"pencil-puzzle-bench-direct-ask-ppbench","signalToNoise":10.532,"beta":4.308,"sigma":0.409,"communality":0.494,"difficulty":168.8,"nModels":49,"precision":2.4,"precisionMidpoint":2.4,"signalPercentile":80},{"slug":"hle","name":"Humanity's Last Exam","family":"hle","signalToNoise":10.522,"beta":2.565,"sigma":0.244,"communality":0.896,"difficulty":163.5,"nModels":130,"precision":2.5,"precisionMidpoint":2.4,"signalPercentile":80},{"slug":"dtbench","name":"DTBench","family":"dtbench","signalToNoise":10.521,"beta":2.133,"sigma":0.203,"communality":0.924,"difficulty":136.3,"nModels":174,"precision":6.6,"precisionMidpoint":2.4,"signalPercentile":80},{"slug":"browsecomp-mercor-extended","name":"BrowseComp Extended (Mercor held-out)","family":"browsecomp-mercor-extended","signalToNoise":10.52,"beta":4.777,"sigma":0.454,"communality":0.7,"difficulty":165.4,"nModels":35,"precision":2.4,"precisionMidpoint":2.4,"signalPercentile":80},{"slug":"emb-v1-1","name":"Excel Modeling Benchmark v1.1","family":"emb-v1-1","signalToNoise":10.469,"beta":1.745,"sigma":0.167,"communality":0.774,"difficulty":149.9,"nModels":36,"precision":2.8,"precisionMidpoint":2.4,"signalPercentile":80},{"slug":"mysterymechanism","name":"MysteryMechanism","family":"mysterymechanism","signalToNoise":10.411,"beta":2.899,"sigma":0.279,"communality":0.695,"difficulty":170.6,"nModels":27,"precision":2.4,"precisionMidpoint":2.4,"signalPercentile":80},{"slug":"reactbench-v1","name":"ReactBench v1","family":"reactbench-v1","signalToNoise":10.404,"beta":2.781,"sigma":0.267,"communality":0.703,"difficulty":165.4,"nModels":16,"precision":2.4,"precisionMidpoint":2.4,"signalPercentile":79},{"slug":"include","name":"INCLUDE","family":"include","signalToNoise":10.388,"beta":1.418,"sigma":0.136,"communality":0.937,"difficulty":126.7,"nModels":27,"precision":5.0,"precisionMidpoint":2.4,"signalPercentile":79},{"slug":"code-migration","name":"Code Migration","family":"code-migration","signalToNoise":10.328,"beta":3.807,"sigma":0.369,"communality":0.593,"difficulty":163.0,"nModels":72,"precision":2.6,"precisionMidpoint":2.4,"signalPercentile":79},{"slug":"nextjs-ai-agent-evals","name":"Next.js AI Agent Evaluations","family":"nextjs-ai-agent-evals","signalToNoise":10.294,"beta":1.915,"sigma":0.186,"communality":0.501,"difficulty":118.2,"nModels":15,"precision":8.4,"precisionMidpoint":2.4,"signalPercentile":79},{"slug":"the-agent-company","name":"The Agent Company","family":"the-agent-company","signalToNoise":10.233,"beta":2.557,"sigma":0.25,"communality":0.895,"difficulty":145.2,"nModels":13,"precision":4.2,"precisionMidpoint":2.5,"signalPercentile":79},{"slug":"bankertoolbench","name":"BankerToolBench","family":"bankertoolbench","signalToNoise":10.22,"beta":2.003,"sigma":0.196,"communality":0.851,"difficulty":153.3,"nModels":9,"precision":2.9,"precisionMidpoint":2.5,"signalPercentile":79},{"slug":"mmlu-pro-openllm-harness","name":"MMLU-Pro (Open LLM Leaderboard v2)","family":"mmlu-pro-openllm-harness","signalToNoise":10.127,"beta":1.39,"sigma":0.137,"communality":0.946,"difficulty":123.6,"nModels":121,"precision":4.9,"precisionMidpoint":2.5,"signalPercentile":79},{"slug":"agents-last-exam-v1-1","name":"Agents' Last Exam v1.1 (ALE v1.1)","family":"agents-last-exam-v1-1","signalToNoise":10.045,"beta":2.068,"sigma":0.206,"communality":0.818,"difficulty":171.5,"nModels":12,"precision":2.5,"precisionMidpoint":2.5,"signalPercentile":79},{"slug":"cybergym","name":"CyberGym","family":"cybergym","signalToNoise":9.995,"beta":4.431,"sigma":0.443,"communality":0.846,"difficulty":151.4,"nModels":43,"precision":5.4,"precisionMidpoint":2.5,"signalPercentile":78},{"slug":"riemannbench","name":"RiemannBench","family":"riemannbench","signalToNoise":9.942,"beta":4.002,"sigma":0.403,"communality":0.829,"difficulty":164.8,"nModels":39,"precision":2.9,"precisionMidpoint":2.5,"signalPercentile":78},{"slug":"cmimc-2025-matharena","name":"CMIMC 2025","family":"cmimc-2025-matharena","signalToNoise":9.933,"beta":2.853,"sigma":0.287,"communality":0.842,"difficulty":136.5,"nModels":33,"precision":10.6,"precisionMidpoint":2.5,"signalPercentile":78},{"slug":"smt-2025-matharena","name":"SMT 2025","family":"smt-2025-matharena","signalToNoise":9.909,"beta":1.928,"sigma":0.195,"communality":0.883,"difficulty":125.2,"nModels":40,"precision":8.1,"precisionMidpoint":2.5,"signalPercentile":78},{"slug":"frontiermath-tiers-1-3-v2-private","name":"FrontierMath Tiers 1-3 (v2) - Private","family":"frontiermath-tiers-1-3-v2-private","signalToNoise":9.904,"beta":3.496,"sigma":0.353,"communality":0.904,"difficulty":152.8,"nModels":84,"precision":4.1,"precisionMidpoint":2.5,"signalPercentile":78},{"slug":"gdp-xlsx","name":"GDP.xlsx","family":"gdp-xlsx","signalToNoise":9.9,"beta":2.197,"sigma":0.222,"communality":0.534,"difficulty":179.0,"nModels":16,"precision":2.9,"precisionMidpoint":2.5,"signalPercentile":78},{"slug":"spatialbench-v2","name":"SpatialBench V1 Clean Reruns","family":"spatialbench-v2","signalToNoise":9.893,"beta":1.497,"sigma":0.151,"communality":0.662,"difficulty":150.9,"nModels":24,"precision":2.9,"precisionMidpoint":2.5,"signalPercentile":78},{"slug":"terminal-bench-2-0","name":"Terminal-Bench 2.0","family":"terminal-bench-2-0","signalToNoise":9.88,"beta":3.011,"sigma":0.305,"communality":0.856,"difficulty":150.5,"nModels":87,"precision":3.9,"precisionMidpoint":2.5,"signalPercentile":77},{"slug":"frontier-financial-judgement","name":"Frontier Financial Judgement","family":"frontier-financial-judgement","signalToNoise":9.871,"beta":2.32,"sigma":0.235,"communality":0.829,"difficulty":160.1,"nModels":13,"precision":2.7,"precisionMidpoint":2.6,"signalPercentile":77},{"slug":"magi-hard","name":"MAGI-Hard","family":"magi-hard","signalToNoise":9.82,"beta":2.133,"sigma":0.217,"communality":0.912,"difficulty":115.6,"nModels":92,"precision":14.0,"precisionMidpoint":2.6,"signalPercentile":77},{"slug":"we-math-testmini-strict","name":"We-Math (testmini, strict score)","family":"we-math-testmini-strict","signalToNoise":9.789,"beta":1.879,"sigma":0.192,"communality":0.974,"difficulty":127.6,"nModels":8,"precision":7.2,"precisionMidpoint":2.6,"signalPercentile":77},{"slug":"frontiercua","name":"FrontierCUA pass@1","family":"frontiercua","signalToNoise":9.779,"beta":3.603,"sigma":0.368,"communality":0.146,"difficulty":171.8,"nModels":13,"precision":2.6,"precisionMidpoint":2.6,"signalPercentile":77},{"slug":"concept-synth-induction-challenge100-round1","name":"Concept Synth INDUCTION Challenge100 Round 1","family":"concept-synth-induction-challenge100-round1","signalToNoise":9.747,"beta":4.968,"sigma":0.51,"communality":0.795,"difficulty":168.0,"nModels":28,"precision":2.6,"precisionMidpoint":2.6,"signalPercentile":77},{"slug":"dual-arm-sim","name":"Dual-Arm Sim (task completion)","family":"dual-arm-sim","signalToNoise":9.738,"beta":3.267,"sigma":0.335,"communality":0.057,"difficulty":194.2,"nModels":10,"precision":7.2,"precisionMidpoint":2.6,"signalPercentile":77},{"slug":"aime-2026-matharena","name":"AIME 2026","family":"aime-2026-matharena","signalToNoise":9.731,"beta":2.84,"sigma":0.292,"communality":0.618,"difficulty":127.2,"nModels":58,"precision":15.1,"precisionMidpoint":2.6,"signalPercentile":77},{"slug":"medxpertqa-mm","name":"MedXpertQA MM","family":"medxpertqa-mm","signalToNoise":9.719,"beta":2.421,"sigma":0.249,"communality":0.911,"difficulty":147.8,"nModels":40,"precision":4.5,"precisionMidpoint":2.6,"signalPercentile":76},{"slug":"live-bench-2","name":"LiveBench 2.0","family":"live-bench-2","signalToNoise":9.718,"beta":2.289,"sigma":0.236,"communality":0.84,"difficulty":150.8,"nModels":53,"precision":3.4,"precisionMidpoint":2.6,"signalPercentile":76},{"slug":"exploitbench-leaderboard-mean-capability-score-out-of-16-exploitbench","name":"ExploitBench Leaderboard (Mean capability score, out of 16)","family":"exploitbench-leaderboard-mean-capability-score-out-of-16-exploitbench","signalToNoise":9.717,"beta":3.342,"sigma":0.344,"communality":0.803,"difficulty":163.6,"nModels":13,"precision":2.7,"precisionMidpoint":2.6,"signalPercentile":76},{"slug":"cad-arena","name":"CAD Arena","family":"cad-arena","signalToNoise":9.687,"beta":3.812,"sigma":0.393,"communality":0.801,"difficulty":164.0,"nModels":12,"precision":2.7,"precisionMidpoint":2.6,"signalPercentile":76},{"slug":"mirrorcode-ml-private-2l","name":"MirrorCode (ML, +Private, 2L)","family":"mirrorcode-ml-private-2l","signalToNoise":9.677,"beta":4.721,"sigma":0.488,"communality":0.764,"difficulty":164.3,"nModels":15,"precision":2.7,"precisionMidpoint":2.6,"signalPercentile":76},{"slug":"variantbench-v2","name":"VariantBench V1 Clean Reruns","family":"variantbench-v2","signalToNoise":9.584,"beta":2.182,"sigma":0.228,"communality":0.617,"difficulty":167.7,"nModels":24,"precision":2.6,"precisionMidpoint":2.6,"signalPercentile":76},{"slug":"apex-agents","name":"APEX-Agents","family":"apex-agents","signalToNoise":9.575,"beta":1.865,"sigma":0.195,"communality":0.749,"difficulty":167.7,"nModels":19,"precision":2.6,"precisionMidpoint":2.6,"signalPercentile":76},{"slug":"mmlu-prox","name":"MMLU-ProX","family":"mmlu-prox","signalToNoise":9.563,"beta":1.313,"sigma":0.137,"communality":0.934,"difficulty":120.8,"nModels":54,"precision":5.2,"precisionMidpoint":2.6,"signalPercentile":75},{"slug":"minebench-bradley-terry-rating","name":"MineBench \u2014 Bradley-Terry Rating","family":"minebench-bradley-terry-rating","signalToNoise":9.556,"beta":2.762,"sigma":0.289,"communality":0.931,"difficulty":154.4,"nModels":72,"precision":2.6,"precisionMidpoint":2.6,"signalPercentile":75},{"slug":"drugdiscoverybench","name":"DrugDiscoveryBench","family":"drugdiscoverybench","signalToNoise":9.55,"beta":2.584,"sigma":0.271,"communality":0.798,"difficulty":160.9,"nModels":18,"precision":2.8,"precisionMidpoint":2.6,"signalPercentile":75},{"slug":"physicianbench","name":"PhysicianBench","family":"physicianbench","signalToNoise":9.527,"beta":3.511,"sigma":0.369,"communality":0.656,"difficulty":164.7,"nModels":13,"precision":2.7,"precisionMidpoint":2.6,"signalPercentile":75},{"slug":"swe-bench-pro-private-dataset-scale-ai-2","name":"SWE-Bench Pro (Private Dataset)","family":"swe-bench-pro-private-dataset-scale-ai-2","signalToNoise":9.505,"beta":3.058,"sigma":0.322,"communality":0.755,"difficulty":160.9,"nModels":12,"precision":2.9,"precisionMidpoint":2.6,"signalPercentile":75},{"slug":"brumo-2025-matharena","name":"BRUMO 2025","family":"brumo-2025-matharena","signalToNoise":9.494,"beta":2.584,"sigma":0.272,"communality":0.474,"difficulty":124.8,"nModels":42,"precision":16.0,"precisionMidpoint":2.7,"signalPercentile":75},{"slug":"gbench-one-shot-coding-gscore","name":"GBENCH One-shot Coding GScore","family":"gbench-one-shot-coding-gscore","signalToNoise":9.479,"beta":2.334,"sigma":0.246,"communality":0.857,"difficulty":158.2,"nModels":101,"precision":3.0,"precisionMidpoint":2.7,"signalPercentile":75},{"slug":"gpqa-diamond","name":"GPQA Diamond","family":"gpqa-diamond","signalToNoise":9.474,"beta":2.077,"sigma":0.219,"communality":0.937,"difficulty":133.1,"nModels":356,"precision":7.1,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"real-swe-1-1","name":"Real-SWE 1.1","family":"real-swe-1-1","signalToNoise":9.455,"beta":3.115,"sigma":0.329,"communality":0.634,"difficulty":168.6,"nModels":13,"precision":2.7,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"mbpp-evalplus-base","name":"MBPP EvalPlus (base tests)","family":"mbpp-evalplus-base","signalToNoise":9.435,"beta":1.327,"sigma":0.141,"communality":0.918,"difficulty":88.9,"nModels":10,"precision":11.0,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"spatialbench-benchmarks-bio","name":"SpatialBench","family":"spatialbench-benchmarks-bio","signalToNoise":9.427,"beta":1.48,"sigma":0.157,"communality":0.856,"difficulty":156.2,"nModels":16,"precision":2.8,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"snorkel-workplace-agents","name":"Snorkel WorkplaceAgents","family":"snorkel-workplace-agents","signalToNoise":9.422,"beta":1.368,"sigma":0.145,"communality":0.241,"difficulty":198.2,"nModels":12,"precision":3.6,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"public-benefits-bench-multi-turn","name":"Public Benefits Bench v1 - Multi Turn","family":"public-benefits-bench-multi-turn","signalToNoise":9.402,"beta":1.098,"sigma":0.117,"communality":0.919,"difficulty":153.9,"nModels":13,"precision":2.8,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"mmlu-openllm-v1-harness","name":"MMLU (Open LLM Leaderboard v1)","family":"mmlu-openllm-v1-harness","signalToNoise":9.389,"beta":1.517,"sigma":0.162,"communality":0.955,"difficulty":98.9,"nModels":117,"precision":12.8,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"legal_research","name":"Legal Research Bench","family":"legal_research","signalToNoise":9.34,"beta":2.782,"sigma":0.298,"communality":0.562,"difficulty":164.1,"nModels":77,"precision":2.7,"precisionMidpoint":2.7,"signalPercentile":74},{"slug":"automationbench-aa-1-0-6","name":"AutomationBench-AA 1.0.6","family":"automationbench-aa-1-0-6","signalToNoise":9.332,"beta":3.871,"sigma":0.415,"communality":0.877,"difficulty":160.5,"nModels":146,"precision":3.1,"precisionMidpoint":2.7,"signalPercentile":73},{"slug":"sage-v1-1","name":"SAGE v1.1","family":"sage-v1-1","signalToNoise":9.29,"beta":1.547,"sigma":0.167,"communality":0.79,"difficulty":144.1,"nModels":25,"precision":5.1,"precisionMidpoint":2.7,"signalPercentile":73},{"slug":"graphwalks-bfs-256k","name":"GraphWalks BFS - 256K","family":"graphwalks-bfs-256k","signalToNoise":9.247,"beta":2.869,"sigma":0.31,"communality":0.534,"difficulty":144.9,"nModels":11,"precision":5.3,"precisionMidpoint":2.7,"signalPercentile":73},{"slug":"continual-learning-exam","name":"Continual Learning Exam (Trial 1)","family":"continual-learning-exam","signalToNoise":9.246,"beta":1.805,"sigma":0.195,"communality":0.558,"difficulty":176.4,"nModels":12,"precision":2.8,"precisionMidpoint":2.7,"signalPercentile":73},{"slug":"tax-agent-bench","name":"Tax Agent Bench","family":"tax-agent-bench","signalToNoise":9.233,"beta":2.465,"sigma":0.267,"communality":0.853,"difficulty":155.2,"nModels":71,"precision":3.2,"precisionMidpoint":2.7,"signalPercentile":73},{"slug":"witnessbench-2-0","name":"WitnessBench v2.0","family":"witnessbench-2-0","signalToNoise":9.23,"beta":3.576,"sigma":0.387,"communality":0.732,"difficulty":175.9,"nModels":12,"precision":3.2,"precisionMidpoint":2.7,"signalPercentile":73},{"slug":"sab-research","name":"SAB Research","family":"sab-research","signalToNoise":9.192,"beta":1.834,"sigma":0.2,"communality":0.781,"difficulty":124.3,"nModels":13,"precision":7.1,"precisionMidpoint":2.7,"signalPercentile":73},{"slug":"otelbench","name":"OTelBench","family":"otelbench","signalToNoise":9.19,"beta":2.911,"sigma":0.317,"communality":0.566,"difficulty":166.0,"nModels":14,"precision":2.7,"precisionMidpoint":2.7,"signalPercentile":72},{"slug":"mmlu-redux","name":"MMLU-Redux","family":"mmlu-redux","signalToNoise":9.158,"beta":1.46,"sigma":0.159,"communality":0.961,"difficulty":103.7,"nModels":173,"precision":10.5,"precisionMidpoint":2.7,"signalPercentile":72},{"slug":"zeroeval-crux","name":"ZeroEval CRUX","family":"zeroeval-crux","signalToNoise":9.135,"beta":1.982,"sigma":0.217,"communality":0.94,"difficulty":111.5,"nModels":41,"precision":16.5,"precisionMidpoint":2.8,"signalPercentile":72},{"slug":"spreadsheetbench-2","name":"SpreadsheetBench 2","family":"spreadsheetbench-2","signalToNoise":9.064,"beta":2.021,"sigma":0.223,"communality":0.796,"difficulty":170.4,"nModels":15,"precision":2.8,"precisionMidpoint":2.8,"signalPercentile":72},{"slug":"frontiermath-tier-4-v2-private","name":"FrontierMath Tier 4 (v2) - Private","family":"frontiermath-tier-4-v2-private","signalToNoise":9.043,"beta":5.437,"sigma":0.601,"communality":0.643,"difficulty":160.7,"nModels":69,"precision":4.1,"precisionMidpoint":2.8,"signalPercentile":72},{"slug":"latchbio-singlecellbench-card","name":"scBench","family":"latchbio-singlecellbench-card","signalToNoise":9.038,"beta":1.641,"sigma":0.182,"communality":0.85,"difficulty":155.4,"nModels":19,"precision":3.0,"precisionMidpoint":2.8,"signalPercentile":72},{"slug":"toloka-arena-v1","name":"Toloka Arena (v1)","family":"toloka-arena-v1","signalToNoise":9.031,"beta":3.158,"sigma":0.35,"communality":0.791,"difficulty":161.1,"nModels":33,"precision":3.1,"precisionMidpoint":2.8,"signalPercentile":72},{"slug":"mmmu","name":"MMMU","family":"mmmu","signalToNoise":9.023,"beta":1.181,"sigma":0.131,"communality":0.957,"difficulty":122.6,"nModels":134,"precision":5.4,"precisionMidpoint":2.8,"signalPercentile":72},{"slug":"gdpval-aa-v2-normalized-artificial-analysis","name":"GDPval-AA v2 (normalized)","family":"gdpval-aa-v2-normalized-artificial-analysis","signalToNoise":9.01,"beta":2.975,"sigma":0.33,"communality":0.581,"difficulty":158.7,"nModels":134,"precision":3.2,"precisionMidpoint":2.8,"signalPercentile":71},{"slug":"blueprint-bench-2","name":"Blueprint-Bench 2","family":"blueprint-bench-2","signalToNoise":8.975,"beta":3.315,"sigma":0.369,"communality":0.345,"difficulty":166.9,"nModels":34,"precision":2.8,"precisionMidpoint":2.8,"signalPercentile":71},{"slug":"swe-shift-shifted","name":"SWE-Shift (Shifted)","family":"swe-shift-shifted","signalToNoise":8.943,"beta":1.774,"sigma":0.198,"communality":0.782,"difficulty":189.4,"nModels":16,"precision":3.6,"precisionMidpoint":2.8,"signalPercentile":71},{"slug":"terminal-bench-hard-artificial-analysis","name":"Terminal-Bench Hard","family":"terminal-bench-hard-artificial-analysis","signalToNoise":8.923,"beta":2.68,"sigma":0.3,"communality":0.692,"difficulty":155.3,"nModels":304,"precision":3.4,"precisionMidpoint":2.8,"signalPercentile":71},{"slug":"ramp-accounting-bench","name":"Ramp Accounting Bench","family":"ramp-accounting-bench","signalToNoise":8.792,"beta":1.39,"sigma":0.158,"communality":0.736,"difficulty":166.9,"nModels":16,"precision":2.9,"precisionMidpoint":2.9,"signalPercentile":71},{"slug":"voxelbench-text-rating","name":"VoxelBench Text Rating","family":"voxelbench-text-rating","signalToNoise":8.776,"beta":4.111,"sigma":0.468,"communality":0.807,"difficulty":159.1,"nModels":51,"precision":2.9,"precisionMidpoint":2.9,"signalPercentile":71},{"slug":"tampered-ml","name":"Tampered ML","family":"tampered-ml","signalToNoise":8.738,"beta":2.287,"sigma":0.262,"communality":0.806,"difficulty":156.9,"nModels":22,"precision":4.0,"precisionMidpoint":2.9,"signalPercentile":71},{"slug":"public-benefits-bench-neither","name":"Public Benefits Bench v1 - Neither","family":"public-benefits-bench-neither","signalToNoise":8.725,"beta":1.485,"sigma":0.17,"communality":0.893,"difficulty":159.3,"nModels":13,"precision":3.0,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"terminal-bench-2-1","name":"Terminal-Bench 2.1","family":"terminal-bench-2-1","signalToNoise":8.686,"beta":3.046,"sigma":0.351,"communality":0.84,"difficulty":148.7,"nModels":213,"precision":4.9,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"solo-bench-easy","name":"SOLO Bench Easy","family":"solo-bench-easy","signalToNoise":8.664,"beta":4.078,"sigma":0.471,"communality":0.332,"difficulty":145.8,"nModels":21,"precision":9.6,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"autoformbench","name":"AutoformBench","family":"autoformbench","signalToNoise":8.65,"beta":2.458,"sigma":0.284,"communality":0.739,"difficulty":161.2,"nModels":9,"precision":3.2,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"thematic-generalization-v2","name":"Thematic Generalization V2","family":"thematic-generalization-v2","signalToNoise":8.592,"beta":2.688,"sigma":0.313,"communality":0.848,"difficulty":140.0,"nModels":23,"precision":8.0,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"multinrc-scale-ai-2","name":"MultiNRC","family":"multinrc-scale-ai-2","signalToNoise":8.581,"beta":2.509,"sigma":0.292,"communality":0.844,"difficulty":152.5,"nModels":37,"precision":3.9,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"hebbia-financial-services-benchmark","name":"Hebbia Financial Services Benchmark","family":"hebbia-financial-services-benchmark","signalToNoise":8.562,"beta":2.118,"sigma":0.247,"communality":0.772,"difficulty":156.6,"nModels":10,"precision":3.3,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"kernelbench-cuda-normalized-rtx-pro-6000","name":"KernelBench CUDA Normalized (RTX PRO 6000)","family":"kernelbench-cuda-normalized-rtx-pro-6000","signalToNoise":8.56,"beta":4.059,"sigma":0.474,"communality":0.546,"difficulty":165.1,"nModels":20,"precision":3.0,"precisionMidpoint":2.9,"signalPercentile":70},{"slug":"mrcr-v2-8-needle-256k-512k","name":"OpenAI MRCR v2 8-needle - 256K-512K","family":"mrcr-v2-8-needle-256k-512k","signalToNoise":8.552,"beta":5.106,"sigma":0.597,"communality":0.348,"difficulty":147.1,"nModels":8,"precision":12.5,"precisionMidpoint":2.9,"signalPercentile":69},{"slug":"browsecomp-zh","name":"BrowseComp-ZH","family":"browsecomp-zh","signalToNoise":8.551,"beta":2.636,"sigma":0.308,"communality":0.719,"difficulty":144.3,"nModels":20,"precision":5.6,"precisionMidpoint":2.9,"signalPercentile":69},{"slug":"skillsbench-without-skills","name":"SkillsBench without Skills","family":"skillsbench-without-skills","signalToNoise":8.545,"beta":2.253,"sigma":0.264,"communality":0.774,"difficulty":163.4,"nModels":45,"precision":3.0,"precisionMidpoint":2.9,"signalPercentile":69},{"slug":"vibe-eval-gemini-judge","name":"Vibe-Eval (Gemini judge)","family":"vibe-eval-gemini-judge","signalToNoise":8.543,"beta":0.876,"sigma":0.103,"communality":0.931,"difficulty":124.4,"nModels":12,"precision":3.8,"precisionMidpoint":2.9,"signalPercentile":69},{"slug":"interactivebench","name":"InteractiveBench (task pass)","family":"interactivebench","signalToNoise":8.531,"beta":1.612,"sigma":0.189,"communality":0.65,"difficulty":175.5,"nModels":10,"precision":3.0,"precisionMidpoint":3.0,"signalPercentile":69},{"slug":"swe-atlas-refactoring-scale-ai-2","name":"SWE Atlas - Refactoring","family":"swe-atlas-refactoring-scale-ai-2","signalToNoise":8.53,"beta":2.378,"sigma":0.279,"communality":0.769,"difficulty":162.2,"nModels":18,"precision":3.0,"precisionMidpoint":3.0,"signalPercentile":69},{"slug":"biosecbench-function-v1","name":"BioSecBench-Function V0","family":"biosecbench-function-v1","signalToNoise":8.443,"beta":1.683,"sigma":0.199,"communality":0.536,"difficulty":169.3,"nModels":13,"precision":3.0,"precisionMidpoint":3.0,"signalPercentile":69},{"slug":"lmca","name":"LMCA","family":"lmca","signalToNoise":8.392,"beta":1.564,"sigma":0.186,"communality":0.946,"difficulty":158.2,"nModels":135,"precision":3.1,"precisionMidpoint":3.0,"signalPercentile":68},{"slug":"mt-bench","name":"MT-Bench","family":"mt-bench","signalToNoise":8.378,"beta":1.12,"sigma":0.134,"communality":0.895,"difficulty":76.3,"nModels":41,"precision":11.9,"precisionMidpoint":3.0,"signalPercentile":68},{"slug":"browsecomp","name":"BrowseComp","family":"browsecomp","signalToNoise":8.333,"beta":3.124,"sigma":0.375,"communality":0.83,"difficulty":146.8,"nModels":70,"precision":6.2,"precisionMidpoint":3.0,"signalPercentile":68},{"slug":"aider-polyglot","name":"Aider Polyglot","family":"aider-polyglot","signalToNoise":8.298,"beta":3.344,"sigma":0.403,"communality":0.836,"difficulty":138.1,"nModels":71,"precision":10.7,"precisionMidpoint":3.0,"signalPercentile":68},{"slug":"biosecbench-surveillance-v2","name":"BioSecBench-Surveillance V1 Clean Reruns","family":"biosecbench-surveillance-v2","signalToNoise":8.215,"beta":1.628,"sigma":0.198,"communality":0.563,"difficulty":164.6,"nModels":23,"precision":3.1,"precisionMidpoint":3.1,"signalPercentile":68},{"slug":"build-a-chip-bench-20-tasks","name":"Build-A-Chip-Bench","family":"build-a-chip-bench","signalToNoise":8.209,"beta":4.489,"sigma":0.547,"communality":0.251,"difficulty":183.8,"nModels":8,"precision":6.5,"precisionMidpoint":3.1,"signalPercentile":68},{"slug":"bbh","name":"BBH (BIG-Bench Hard)","family":"bbh","signalToNoise":8.179,"beta":1.875,"sigma":0.229,"communality":0.936,"difficulty":105.1,"nModels":109,"precision":19.0,"precisionMidpoint":3.1,"signalPercentile":68},{"slug":"apex-shortlist-matharena","name":"Apex Shortlist","family":"apex-shortlist-matharena","signalToNoise":8.153,"beta":3.643,"sigma":0.447,"communality":0.786,"difficulty":146.7,"nModels":45,"precision":8.4,"precisionMidpoint":3.1,"signalPercentile":68},{"slug":"immigration-bench","name":"Immigration Bench","family":"immigration-bench","signalToNoise":8.147,"beta":0.913,"sigma":0.112,"communality":0.701,"difficulty":142.6,"nModels":23,"precision":3.4,"precisionMidpoint":3.1,"signalPercentile":67},{"slug":"erdosbench","name":"ErdosBench","family":"erdosbench","signalToNoise":8.121,"beta":1.674,"sigma":0.206,"communality":0.669,"difficulty":146.6,"nModels":22,"precision":3.9,"precisionMidpoint":3.1,"signalPercentile":67},{"slug":"gpqa-main","name":"GPQA Main","family":"gpqa-main","signalToNoise":8.118,"beta":1.924,"sigma":0.237,"communality":0.881,"difficulty":137.7,"nModels":79,"precision":6.6,"precisionMidpoint":3.1,"signalPercentile":67},{"slug":"txbench-od-v1","name":"TxBench Oligo Discovery V0","family":"txbench-od-v1","signalToNoise":8.108,"beta":1.787,"sigma":0.22,"communality":0.505,"difficulty":168.7,"nModels":13,"precision":3.1,"precisionMidpoint":3.1,"signalPercentile":67},{"slug":"deepsecbench-revalidated-v1-233","name":"DeepsecBench","family":"deepsecbench","signalToNoise":8.095,"beta":2.724,"sigma":0.337,"communality":0.648,"difficulty":174.4,"nModels":28,"precision":3.3,"precisionMidpoint":3.1,"signalPercentile":67},{"slug":"emb","name":"Excel Modeling Benchmark","family":"emb","signalToNoise":8.023,"beta":2.689,"sigma":0.335,"communality":0.806,"difficulty":155.3,"nModels":73,"precision":3.8,"precisionMidpoint":3.1,"signalPercentile":67},{"slug":"swe-clar-implementation","name":"SWE-Clar (implementation)","family":"swe-clar-implementation","signalToNoise":7.982,"beta":1.857,"sigma":0.233,"communality":0.659,"difficulty":158.6,"nModels":8,"precision":3.4,"precisionMidpoint":3.2,"signalPercentile":67},{"slug":"imo-proofbench-advanced-usamo-2025","name":"IMO-ProofBench Advanced: USAMO 2025","family":"imo-proofbench-advanced-usamo-2025","signalToNoise":7.978,"beta":3.961,"sigma":0.496,"communality":0.423,"difficulty":154.7,"nModels":22,"precision":5.2,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"chartography-surge-ai-2","name":"Chartography","family":"chartography-surge-ai-2","signalToNoise":7.912,"beta":3.318,"sigma":0.419,"communality":0.668,"difficulty":166.7,"nModels":40,"precision":3.2,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"senior-swe-bench","name":"Senior SWE-Bench","family":"senior-swe-bench","signalToNoise":7.91,"beta":2.815,"sigma":0.356,"communality":0.45,"difficulty":171.7,"nModels":23,"precision":3.3,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"aa-analyst-agent-pass-caret-5","name":"AA-AnalystAgent (pass^5)","family":"aa-analyst-agent-pass-caret-5","signalToNoise":7.9,"beta":2.676,"sigma":0.339,"communality":0.807,"difficulty":164.6,"nModels":48,"precision":3.2,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"swe-bench-pro-v2","name":"SWE-Bench Pro V2 \u2014 Full","family":"swe-bench-pro-v2","signalToNoise":7.889,"beta":2.052,"sigma":0.26,"communality":0.34,"difficulty":121.8,"nModels":16,"precision":10.9,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"cvdp-517","name":"CVDP (517-problem set)","family":"cvdp-517","signalToNoise":7.857,"beta":0.977,"sigma":0.124,"communality":0.649,"difficulty":156.4,"nModels":22,"precision":3.3,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"librarydesignbench","name":"LibraryDesignBench","family":"librarydesignbench","signalToNoise":7.848,"beta":1.072,"sigma":0.137,"communality":0.588,"difficulty":171.0,"nModels":8,"precision":3.2,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"realm-financial","name":"Realm Financial Reasoning","family":"realm-financial","signalToNoise":7.845,"beta":0.8,"sigma":0.102,"communality":0.702,"difficulty":167.8,"nModels":17,"precision":3.2,"precisionMidpoint":3.2,"signalPercentile":66},{"slug":"cwe-bench-v1","name":"CWE-bench v1 Programmatic Pass@1","family":"cwe-bench-v1","signalToNoise":7.84,"beta":1.607,"sigma":0.205,"communality":0.795,"difficulty":156.4,"nModels":12,"precision":3.4,"precisionMidpoint":3.2,"signalPercentile":65},{"slug":"swe-marathon-v1-1","name":"SWE-Marathon v1.1","family":"swe-marathon-v1-1","signalToNoise":7.838,"beta":3.328,"sigma":0.425,"communality":0.408,"difficulty":167.1,"nModels":26,"precision":3.2,"precisionMidpoint":3.2,"signalPercentile":65},{"slug":"cruxeval-i","name":"CRUXEval-I","family":"cruxeval-i","signalToNoise":7.818,"beta":1.526,"sigma":0.195,"communality":0.854,"difficulty":110.9,"nModels":21,"precision":9.5,"precisionMidpoint":3.2,"signalPercentile":65},{"slug":"snorkel-revops","name":"SnorkelRevOps","family":"snorkel-revops","signalToNoise":7.774,"beta":1.636,"sigma":0.21,"communality":0.143,"difficulty":195.9,"nModels":13,"precision":4.6,"precisionMidpoint":3.2,"signalPercentile":65},{"slug":"textquests-game-progress","name":"TextQuests Game Progress","family":"textquests-game-progress","signalToNoise":7.774,"beta":2.208,"sigma":0.284,"communality":0.822,"difficulty":162.9,"nModels":62,"precision":3.4,"precisionMidpoint":3.2,"signalPercentile":65},{"slug":"apex-agents-1-1","name":"APEX-Agents 1.1","family":"apex-agents-1-1","signalToNoise":7.757,"beta":2.385,"sigma":0.307,"communality":0.737,"difficulty":153.2,"nModels":58,"precision":4.0,"precisionMidpoint":3.2,"signalPercentile":65},{"slug":"scicode","name":"SciCode","family":"scicode","signalToNoise":7.731,"beta":1.178,"sigma":0.152,"communality":0.936,"difficulty":155.0,"nModels":39,"precision":3.4,"precisionMidpoint":3.3,"signalPercentile":65},{"slug":"apex-accounting-1-1","name":"APEX-Accounting 1.1","family":"apex-accounting-1-1","signalToNoise":7.683,"beta":2.409,"sigma":0.314,"communality":0.635,"difficulty":158.5,"nModels":43,"precision":3.6,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"apex-swe-mercor-2","name":"APEX-SWE","family":"apex-swe-mercor-2","signalToNoise":7.642,"beta":2.011,"sigma":0.263,"communality":0.755,"difficulty":162.5,"nModels":62,"precision":3.4,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"hmmt-nov-2025-matharena","name":"HMMT Nov 2025","family":"hmmt-nov-2025-matharena","signalToNoise":7.641,"beta":2.662,"sigma":0.348,"communality":0.501,"difficulty":126.5,"nModels":45,"precision":14.9,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"webdev-arena","name":"LMArena Code Arena WebDev","family":"webdev-arena","signalToNoise":7.631,"beta":2.759,"sigma":0.362,"communality":0.881,"difficulty":153.6,"nModels":111,"precision":3.3,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"westworld-finance-diligence","name":"Westworld Finance Diligence Bench","family":"westworld-finance-diligence","signalToNoise":7.597,"beta":2.521,"sigma":0.332,"communality":0.538,"difficulty":165.6,"nModels":9,"precision":3.3,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"epibench-v2","name":"EpiBench V1 Clean Reruns","family":"epibench-v2","signalToNoise":7.572,"beta":1.519,"sigma":0.201,"communality":0.407,"difficulty":182.4,"nModels":24,"precision":3.6,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"witnessbench-1-0","name":"WitnessBench v1.0","family":"witnessbench-1-0","signalToNoise":7.552,"beta":2.764,"sigma":0.366,"communality":0.766,"difficulty":177.3,"nModels":17,"precision":3.7,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"psebench","name":"PSEBench","family":"psebench","signalToNoise":7.527,"beta":1.976,"sigma":0.263,"communality":0.902,"difficulty":130.4,"nModels":12,"precision":9.6,"precisionMidpoint":3.3,"signalPercentile":64},{"slug":"mcp-atlas-april-2026-all-1000","name":"MCP Atlas April 2026 (all 1,000 tasks)","family":"mcp-atlas-april-2026-all-1000","signalToNoise":7.522,"beta":2.356,"sigma":0.313,"communality":0.666,"difficulty":143.2,"nModels":36,"precision":5.7,"precisionMidpoint":3.3,"signalPercentile":63},{"slug":"sci-agent-exam","name":"SCI-AGENT-EXAM","family":"sci-agent-exam","signalToNoise":7.509,"beta":1.633,"sigma":0.217,"communality":0.735,"difficulty":153.2,"nModels":10,"precision":3.8,"precisionMidpoint":3.4,"signalPercentile":63},{"slug":"finance-agent-v2","name":"Finance Agent v2","family":"finance-agent-v2","signalToNoise":7.508,"beta":1.566,"sigma":0.209,"communality":0.816,"difficulty":157.4,"nModels":75,"precision":3.5,"precisionMidpoint":3.4,"signalPercentile":63},{"slug":"mindcube-tiny","name":"MindCube","family":"mindcube","signalToNoise":7.498,"beta":2.517,"sigma":0.336,"communality":0.792,"difficulty":152.2,"nModels":56,"precision":6.1,"precisionMidpoint":3.4,"signalPercentile":63},{"slug":"toolathlon","name":"Toolathlon","family":"toolathlon","signalToNoise":7.496,"beta":2.497,"sigma":0.333,"communality":0.679,"difficulty":156.8,"nModels":41,"precision":3.8,"precisionMidpoint":3.4,"signalPercentile":63},{"slug":"deepplanning","name":"DeepPlanning","family":"deepplanning","signalToNoise":7.476,"beta":3.094,"sigma":0.414,"communality":0.528,"difficulty":157.5,"nModels":23,"precision":4.0,"precisionMidpoint":3.4,"signalPercentile":63},{"slug":"weirdml","name":"WeirdML (v2)","family":"weirdml","signalToNoise":7.449,"beta":2.547,"sigma":0.342,"communality":0.854,"difficulty":151.0,"nModels":119,"precision":5.2,"precisionMidpoint":3.4,"signalPercentile":63},{"slug":"siqa-olmo3-mc","name":"Social IQa (OLMo 3 5-shot multiple choice)","family":"siqa-olmo3-mc","signalToNoise":7.445,"beta":0.788,"sigma":0.106,"communality":0.742,"difficulty":79.9,"nModels":9,"precision":8.9,"precisionMidpoint":3.4,"signalPercentile":62},{"slug":"drone-bench","name":"Drone-Bench","family":"drone-bench","signalToNoise":7.365,"beta":2.732,"sigma":0.371,"communality":0.834,"difficulty":148.2,"nModels":18,"precision":5.7,"precisionMidpoint":3.4,"signalPercentile":62},{"slug":"hle-with-tools","name":"Humanity's Last Exam (with tools)","family":"hle-with-tools","signalToNoise":7.364,"beta":1.718,"sigma":0.233,"communality":0.782,"difficulty":156.9,"nModels":73,"precision":3.7,"precisionMidpoint":3.4,"signalPercentile":62},{"slug":"aa-lcr-mercor-extended","name":"AA-LCR Extended (Mercor held-out)","family":"aa-lcr-mercor-extended","signalToNoise":7.346,"beta":2.23,"sigma":0.304,"communality":0.774,"difficulty":154.2,"nModels":46,"precision":4.1,"precisionMidpoint":3.4,"signalPercentile":62},{"slug":"deepsearchqa-f1-score","name":"DeepSearchQA (F1 score)","family":"deepsearchqa-f1-score","signalToNoise":7.328,"beta":3.234,"sigma":0.441,"communality":0.75,"difficulty":144.2,"nModels":29,"precision":8.1,"precisionMidpoint":3.4,"signalPercentile":62},{"slug":"ahk-eval","name":"AHK-Eval","family":"ahk-eval","signalToNoise":7.258,"beta":3.1,"sigma":0.427,"communality":0.465,"difficulty":144.0,"nModels":34,"precision":8.1,"precisionMidpoint":3.5,"signalPercentile":62},{"slug":"webarena-verified","name":"WebArena-Verified","family":"webarena-verified","signalToNoise":7.244,"beta":2.256,"sigma":0.311,"communality":0.889,"difficulty":151.2,"nModels":12,"precision":4.5,"precisionMidpoint":3.5,"signalPercentile":62},{"slug":"llm-debate-benchmark","name":"LLM Debate Benchmark","family":"llm-debate-benchmark","signalToNoise":7.241,"beta":2.772,"sigma":0.383,"communality":0.874,"difficulty":154.0,"nModels":50,"precision":3.5,"precisionMidpoint":3.5,"signalPercentile":62},{"slug":"graphwalks-bfs-1m","name":"GraphWalks BFS - 1M","family":"graphwalks-bfs-1m","signalToNoise":7.189,"beta":4.39,"sigma":0.611,"communality":0.441,"difficulty":158.5,"nModels":10,"precision":4.6,"precisionMidpoint":3.5,"signalPercentile":61},{"slug":"swe-bench-pro-public-dataset-scale-ai-2","name":"SWE-Bench Pro (Public Dataset)","family":"swe-bench-pro-public-dataset-scale-ai-2","signalToNoise":7.174,"beta":1.89,"sigma":0.263,"communality":0.825,"difficulty":151.0,"nModels":70,"precision":4.2,"precisionMidpoint":3.5,"signalPercentile":61},{"slug":"long-horizon-terminal-bench-1-0","name":"Long-Horizon Terminal-Bench 1.0 (mean reward)","family":"long-horizon-terminal-bench-1-0","signalToNoise":7.17,"beta":1.924,"sigma":0.268,"communality":0.731,"difficulty":164.4,"nModels":21,"precision":3.6,"precisionMidpoint":3.5,"signalPercentile":61},{"slug":"complexconstraints-surge-ai-2","name":"ComplexConstraints","family":"complexconstraints-surge-ai-2","signalToNoise":7.153,"beta":2.209,"sigma":0.309,"communality":0.566,"difficulty":165.4,"nModels":37,"precision":3.6,"precisionMidpoint":3.5,"signalPercentile":61},{"slug":"cybench","name":"Cybench","family":"cybench","signalToNoise":7.115,"beta":2.913,"sigma":0.409,"communality":0.783,"difficulty":145.4,"nModels":13,"precision":6.8,"precisionMidpoint":3.5,"signalPercentile":61},{"slug":"global-piqa","name":"Global PIQA","family":"global-piqa","signalToNoise":7.028,"beta":1.048,"sigma":0.149,"communality":0.863,"difficulty":98.6,"nModels":33,"precision":8.0,"precisionMidpoint":3.6,"signalPercentile":61},{"slug":"gorilla-apibench","name":"Gorilla APIBench","family":"gorilla-apibench","signalToNoise":7.012,"beta":2.517,"sigma":0.359,"communality":0.794,"difficulty":132.1,"nModels":9,"precision":11.3,"precisionMidpoint":3.6,"signalPercentile":61},{"slug":"gba-eval-overall-gbaeval","name":"GBA Eval Overall","family":"gba-eval-overall-gbaeval","signalToNoise":6.969,"beta":6.49,"sigma":0.931,"communality":0.385,"difficulty":161.8,"nModels":22,"precision":4.7,"precisionMidpoint":3.6,"signalPercentile":60},{"slug":"astabench-2026-paperfinding","name":"PaperFindingBench (AstaBench 2026)","family":"astabench-2026-paperfinding","signalToNoise":6.956,"beta":1.135,"sigma":0.163,"communality":0.915,"difficulty":173.2,"nModels":13,"precision":3.6,"precisionMidpoint":3.6,"signalPercentile":60},{"slug":"longbench-v2","name":"LongBench v2","family":"longbench-v2","signalToNoise":6.955,"beta":1.307,"sigma":0.188,"communality":0.837,"difficulty":152.3,"nModels":50,"precision":4.7,"precisionMidpoint":3.6,"signalPercentile":60},{"slug":"swe-bench-multimodal-vals-v1","name":"SWE-bench Multimodal","family":"swe-bench-multimodal","signalToNoise":6.937,"beta":1.716,"sigma":0.247,"communality":0.612,"difficulty":165.2,"nModels":19,"precision":3.6,"precisionMidpoint":3.6,"signalPercentile":60},{"slug":"openbookqa-helm-lite","name":"OpenBookQA (HELM Lite, 5-shot multiple choice)","family":"openbookqa-helm-lite","signalToNoise":6.925,"beta":1.58,"sigma":0.228,"communality":0.827,"difficulty":85.3,"nModels":72,"precision":28.7,"precisionMidpoint":3.6,"signalPercentile":60},{"slug":"arena-hard-v2","name":"Arena-Hard-Auto v2.0","family":"arena-hard-v2","signalToNoise":6.923,"beta":2.687,"sigma":0.388,"communality":0.873,"difficulty":135.1,"nModels":40,"precision":10.9,"precisionMidpoint":3.6,"signalPercentile":60},{"slug":"medxpertqa-text","name":"MedXpertQA Text","family":"medxpertqa-text","signalToNoise":6.923,"beta":1.737,"sigma":0.251,"communality":0.932,"difficulty":153.9,"nModels":14,"precision":4.4,"precisionMidpoint":3.6,"signalPercentile":60},{"slug":"arc-agi-3","name":"ARC-AGI-3 (Semi-Private)","family":"arc-agi-3","signalToNoise":6.896,"beta":5.711,"sigma":0.828,"communality":0.473,"difficulty":171.4,"nModels":25,"precision":3.9,"precisionMidpoint":3.7,"signalPercentile":60},{"slug":"mmlu-pro","name":"MMLU Pro","family":"mmlu-pro","signalToNoise":6.89,"beta":1.388,"sigma":0.201,"communality":0.938,"difficulty":116.7,"nModels":432,"precision":8.5,"precisionMidpoint":3.7,"signalPercentile":59},{"slug":"mmmu-pro","name":"MMMU Pro","family":"mmmu-pro","signalToNoise":6.864,"beta":1.21,"sigma":0.176,"communality":0.919,"difficulty":132.4,"nModels":189,"precision":5.5,"precisionMidpoint":3.7,"signalPercentile":59},{"slug":"formationeval","name":"FormationEval","family":"formationeval","signalToNoise":6.857,"beta":1.581,"sigma":0.231,"communality":0.689,"difficulty":100.9,"nModels":57,"precision":17.6,"precisionMidpoint":3.7,"signalPercentile":59},{"slug":"math-vision","name":"MATH-Vision","family":"math-vision","signalToNoise":6.823,"beta":2.02,"sigma":0.296,"communality":0.919,"difficulty":134.3,"nModels":100,"precision":8.5,"precisionMidpoint":3.7,"signalPercentile":59},{"slug":"kernelbench-mega-normalized-h100","name":"KernelBench Mega Normalized (H100)","family":"kernelbench-mega-normalized-h100","signalToNoise":6.797,"beta":4.899,"sigma":0.721,"communality":0.209,"difficulty":167.9,"nModels":15,"precision":3.7,"precisionMidpoint":3.7,"signalPercentile":59},{"slug":"arena-hard-v0-1","name":"Arena-Hard-Auto v0.1","family":"arena-hard-v0-1","signalToNoise":6.789,"beta":2.731,"sigma":0.402,"communality":0.868,"difficulty":115.5,"nModels":65,"precision":30.8,"precisionMidpoint":3.7,"signalPercentile":59},{"slug":"snorkel-agentic-coding","name":"Snorkel Agentic Coding","family":"snorkel-agentic-coding","signalToNoise":6.777,"beta":2.314,"sigma":0.341,"communality":0.777,"difficulty":151.3,"nModels":11,"precision":4.8,"precisionMidpoint":3.7,"signalPercentile":59},{"slug":"txbench-ab-v1","name":"TxBench Antibody Discovery V0","family":"txbench-ab-v1","signalToNoise":6.76,"beta":2.146,"sigma":0.317,"communality":0.43,"difficulty":166.2,"nModels":14,"precision":3.7,"precisionMidpoint":3.7,"signalPercentile":58},{"slug":"osworld-verified","name":"OSWorld","family":"os-world","signalToNoise":6.718,"beta":2.631,"sigma":0.392,"communality":0.803,"difficulty":147.2,"nModels":53,"precision":6.3,"precisionMidpoint":3.7,"signalPercentile":58},{"slug":"gsm1k-lm-eval-harness","name":"GSM1k (paper LM Evaluation Harness protocol)","family":"gsm1k-lm-eval-harness","signalToNoise":6.698,"beta":2.299,"sigma":0.343,"communality":0.908,"difficulty":100.3,"nModels":46,"precision":42.2,"precisionMidpoint":3.8,"signalPercentile":58},{"slug":"global-mmlu-lite","name":"Global MMLU Lite","family":"global-mmlu-lite","signalToNoise":6.683,"beta":1.367,"sigma":0.204,"communality":0.854,"difficulty":118.9,"nModels":93,"precision":9.0,"precisionMidpoint":3.8,"signalPercentile":58},{"slug":"dynamath","name":"DynaMath","family":"dynamath","signalToNoise":6.679,"beta":1.06,"sigma":0.159,"communality":0.897,"difficulty":107.2,"nModels":29,"precision":7.6,"precisionMidpoint":3.8,"signalPercentile":58},{"slug":"jobbench-main","name":"JobBench Main","family":"jobbench-main","signalToNoise":6.668,"beta":2.781,"sigma":0.417,"communality":0.601,"difficulty":160.3,"nModels":22,"precision":4.1,"precisionMidpoint":3.8,"signalPercentile":58},{"slug":"biomysterybench-human-solvable","name":"BioMysteryBench - Human Solvable","family":"biomysterybench-human-solvable","signalToNoise":6.653,"beta":1.404,"sigma":0.211,"communality":0.193,"difficulty":123.2,"nModels":9,"precision":7.2,"precisionMidpoint":3.8,"signalPercentile":58},{"slug":"scbench-long-v1-verified","name":"ScBench-Long V0 Verified","family":"scbench-long-v1-verified","signalToNoise":6.652,"beta":2.092,"sigma":0.314,"communality":0.327,"difficulty":175.0,"nModels":15,"precision":3.9,"precisionMidpoint":3.8,"signalPercentile":58},{"slug":"finance-agent-v2-1","name":"Finance Agent v2.1","family":"finance-agent-v2-1","signalToNoise":6.65,"beta":0.969,"sigma":0.146,"communality":0.588,"difficulty":149.1,"nModels":32,"precision":4.0,"precisionMidpoint":3.8,"signalPercentile":57},{"slug":"hlab-contracts","name":"Harvey LAB: In-House Contracting (held-out)","family":"hlab-contracts","signalToNoise":6.648,"beta":2.366,"sigma":0.356,"communality":0.046,"difficulty":199.4,"nModels":13,"precision":8.6,"precisionMidpoint":3.8,"signalPercentile":57},{"slug":"healthbench-professional","name":"HealthBench Professional","family":"healthbench-professional","signalToNoise":6.639,"beta":1.154,"sigma":0.174,"communality":0.775,"difficulty":151.4,"nModels":30,"precision":4.1,"precisionMidpoint":3.8,"signalPercentile":57},{"slug":"toloka-arena-v3","name":"Toloka Arena (v3)","family":"toloka-arena-v3","signalToNoise":6.607,"beta":3.684,"sigma":0.558,"communality":0.537,"difficulty":160.5,"nModels":24,"precision":4.4,"precisionMidpoint":3.8,"signalPercentile":57},{"slug":"harvey-lab-mercor","name":"Harvey LAB (Mercor run, 1,749 public tasks)","family":"harvey-lab-mercor","signalToNoise":6.592,"beta":2.627,"sigma":0.398,"communality":0.218,"difficulty":186.7,"nModels":18,"precision":6.0,"precisionMidpoint":3.8,"signalPercentile":57},{"slug":"charxiv-descriptive","name":"CharXiv-D","family":"charxiv-descriptive","signalToNoise":6.528,"beta":1.312,"sigma":0.201,"communality":0.885,"difficulty":100.7,"nModels":64,"precision":12.2,"precisionMidpoint":3.9,"signalPercentile":57},{"slug":"metagenomicsbench-v1","name":"MetagenomicsBench V1","family":"metagenomicsbench-v1","signalToNoise":6.519,"beta":1.964,"sigma":0.301,"communality":0.461,"difficulty":168.5,"nModels":23,"precision":3.9,"precisionMidpoint":3.9,"signalPercentile":57},{"slug":"pwnbench-v0-1","name":"PWNBench v0.1","family":"pwnbench-v0-1","signalToNoise":6.503,"beta":1.591,"sigma":0.245,"communality":0.634,"difficulty":159.1,"nModels":13,"precision":4.0,"precisionMidpoint":3.9,"signalPercentile":56},{"slug":"swe-bench-multilingual","name":"SWE-bench Multilingual","family":"swe-bench-multilingual","signalToNoise":6.408,"beta":2.384,"sigma":0.372,"communality":0.739,"difficulty":141.8,"nModels":92,"precision":7.1,"precisionMidpoint":3.9,"signalPercentile":56},{"slug":"instruction-following-scale-ai-2","name":"Instruction Following","family":"instruction-following-scale-ai-2","signalToNoise":6.399,"beta":0.844,"sigma":0.132,"communality":0.644,"difficulty":77.6,"nModels":19,"precision":9.4,"precisionMidpoint":3.9,"signalPercentile":56},{"slug":"humanitys-sixth-sense","name":"Humanity's Sixth Sense","family":"humanitys-sixth-sense","signalToNoise":6.367,"beta":1.866,"sigma":0.293,"communality":0.464,"difficulty":172.4,"nModels":28,"precision":4.0,"precisionMidpoint":4.0,"signalPercentile":56},{"slug":"clinicare-bench","name":"CliniCARE-Bench","family":"clinicare-bench","signalToNoise":6.364,"beta":0.854,"sigma":0.134,"communality":0.508,"difficulty":141.0,"nModels":12,"precision":5.3,"precisionMidpoint":4.0,"signalPercentile":56},{"slug":"mmlu","name":"MMLU","family":"mmlu","signalToNoise":6.338,"beta":1.192,"sigma":0.188,"communality":0.93,"difficulty":99.8,"nModels":267,"precision":11.9,"precisionMidpoint":4.0,"signalPercentile":56},{"slug":"epibench-benchmarks-bio","name":"EpiBench","family":"epibench-benchmarks-bio","signalToNoise":6.334,"beta":1.569,"sigma":0.248,"communality":0.59,"difficulty":170.7,"nModels":21,"precision":4.0,"precisionMidpoint":4.0,"signalPercentile":56},{"slug":"taxcalcbench-ty24","name":"TaxCalcBench TY24","family":"taxcalcbench-ty24","signalToNoise":6.329,"beta":1.842,"sigma":0.291,"communality":0.76,"difficulty":158.8,"nModels":17,"precision":4.2,"precisionMidpoint":4.0,"signalPercentile":56},{"slug":"public-benefits-bench-web-search","name":"Public Benefits Bench v1 - Web Search","family":"public-benefits-bench-web-search","signalToNoise":6.306,"beta":1.259,"sigma":0.2,"communality":0.776,"difficulty":154.1,"nModels":13,"precision":4.2,"precisionMidpoint":4.0,"signalPercentile":55},{"slug":"simplebench","name":"SimpleBench","family":"simplebench","signalToNoise":6.298,"beta":2.201,"sigma":0.349,"communality":0.828,"difficulty":153.4,"nModels":100,"precision":5.4,"precisionMidpoint":4.0,"signalPercentile":55},{"slug":"banking-artificial-analysis","name":"\ud835\udf0f\u00b3-Banking","family":"banking-artificial-analysis","signalToNoise":6.295,"beta":1.958,"sigma":0.311,"communality":0.777,"difficulty":167.8,"nModels":175,"precision":4.0,"precisionMidpoint":4.0,"signalPercentile":55},{"slug":"apex-accounting","name":"APEX-Accounting","family":"apex-accounting","signalToNoise":6.249,"beta":2.338,"sigma":0.374,"communality":0.677,"difficulty":162.4,"nModels":33,"precision":4.1,"precisionMidpoint":4.0,"signalPercentile":55},{"slug":"zeroeval-zebralogic","name":"ZeroEval ZebraLogic","family":"zeroeval-zebralogic","signalToNoise":6.244,"beta":2.477,"sigma":0.397,"communality":0.854,"difficulty":132.4,"nModels":52,"precision":14.7,"precisionMidpoint":4.0,"signalPercentile":55},{"slug":"ebr-bench-v4-card-ban","name":"EBR-bench v4 (card ban, single agent)","family":"ebr-bench-v4-card-ban","signalToNoise":6.219,"beta":2.592,"sigma":0.417,"communality":0.469,"difficulty":162.8,"nModels":11,"precision":4.3,"precisionMidpoint":4.0,"signalPercentile":55},{"slug":"enterprisebench-corecraft-surge-ai-2","name":"EnterpriseBench: CoreCraft","family":"enterprisebench-corecraft-surge-ai-2","signalToNoise":6.211,"beta":2.765,"sigma":0.445,"communality":0.722,"difficulty":160.7,"nModels":43,"precision":4.3,"precisionMidpoint":4.1,"signalPercentile":55},{"slug":"logicvista","name":"LogicVista","family":"logicvista","signalToNoise":6.21,"beta":1.089,"sigma":0.175,"communality":0.891,"difficulty":125.8,"nModels":26,"precision":6.2,"precisionMidpoint":4.1,"signalPercentile":54},{"slug":"cl-bench","name":"CL-bench (Overall)","family":"cl-bench","signalToNoise":6.207,"beta":1.004,"sigma":0.162,"communality":0.464,"difficulty":188.6,"nModels":23,"precision":4.4,"precisionMidpoint":4.1,"signalPercentile":54},{"slug":"visual-math-overall-matharena","name":"Visual Math \u2014 Overall","family":"visual-math-overall-matharena","signalToNoise":6.185,"beta":1.678,"sigma":0.271,"communality":0.639,"difficulty":133.2,"nModels":22,"precision":8.1,"precisionMidpoint":4.1,"signalPercentile":54},{"slug":"mls-bench-lite","name":"MLS Bench Lite","family":"mls-bench-lite","signalToNoise":6.177,"beta":1.334,"sigma":0.216,"communality":0.559,"difficulty":168.3,"nModels":8,"precision":4.1,"precisionMidpoint":4.1,"signalPercentile":54},{"slug":"alignbench","name":"AlignBench (v1.0 or version unstated)","family":"alignbench","signalToNoise":6.159,"beta":1.108,"sigma":0.18,"communality":0.811,"difficulty":94.0,"nModels":21,"precision":10.9,"precisionMidpoint":4.1,"signalPercentile":54},{"slug":"active-probing-benchmark","name":"Active Probing Benchmark (APB)","family":"active-probing-benchmark","signalToNoise":6.122,"beta":1.822,"sigma":0.298,"communality":0.443,"difficulty":166.1,"nModels":8,"precision":4.1,"precisionMidpoint":4.1,"signalPercentile":54},{"slug":"kimi-code-bench-v2","name":"Kimi Code Bench v2","family":"kimi-code-bench-v2","signalToNoise":6.096,"beta":1.369,"sigma":0.225,"communality":0.606,"difficulty":144.4,"nModels":8,"precision":5.0,"precisionMidpoint":4.1,"signalPercentile":54},{"slug":"erqa","name":"ERQA","family":"erqa","signalToNoise":6.039,"beta":1.343,"sigma":0.222,"communality":0.823,"difficulty":153.7,"nModels":93,"precision":5.5,"precisionMidpoint":4.2,"signalPercentile":54},{"slug":"bedside-bench","name":"Bedside Bench","family":"bedside-bench","signalToNoise":6.025,"beta":1.495,"sigma":0.248,"communality":0.267,"difficulty":123.1,"nModels":15,"precision":8.6,"precisionMidpoint":4.2,"signalPercentile":53},{"slug":"compilebench","name":"CompileBench","family":"compilebench","signalToNoise":5.997,"beta":2.998,"sigma":0.5,"communality":0.541,"difficulty":137.3,"nModels":19,"precision":13.6,"precisionMidpoint":4.2,"signalPercentile":53},{"slug":"cruxeval-o","name":"CRUXEval-O","family":"cruxeval-o","signalToNoise":5.989,"beta":1.783,"sigma":0.298,"communality":0.747,"difficulty":109.5,"nModels":21,"precision":17.2,"precisionMidpoint":4.2,"signalPercentile":53},{"slug":"imo-answerbench","name":"IMO-AnswerBench","family":"imo-answerbench","signalToNoise":5.981,"beta":1.918,"sigma":0.321,"communality":0.662,"difficulty":131.5,"nModels":35,"precision":10.0,"precisionMidpoint":4.2,"signalPercentile":53},{"slug":"polymath","name":"PolyMath","family":"polymath","signalToNoise":5.95,"beta":1.744,"sigma":0.293,"communality":0.88,"difficulty":141.1,"nModels":85,"precision":6.7,"precisionMidpoint":4.2,"signalPercentile":53},{"slug":"opends-world","name":"OpenDS-World","family":"opends-world","signalToNoise":5.947,"beta":1.977,"sigma":0.332,"communality":0.375,"difficulty":177.5,"nModels":9,"precision":4.5,"precisionMidpoint":4.2,"signalPercentile":53},{"slug":"enterpriseops-gym-aa-artificial-analysis","name":"EnterpriseOps-Gym-AA","family":"enterpriseops-gym-aa-artificial-analysis","signalToNoise":5.93,"beta":1.08,"sigma":0.182,"communality":0.693,"difficulty":165.0,"nModels":45,"precision":4.3,"precisionMidpoint":4.2,"signalPercentile":53},{"slug":"epiq-corporatebench","name":"Epiq LAER CorporateBench","family":"epiq-corporatebench","signalToNoise":5.928,"beta":0.853,"sigma":0.144,"communality":0.417,"difficulty":126.0,"nModels":10,"precision":5.4,"precisionMidpoint":4.2,"signalPercentile":52},{"slug":"mmstar","name":"MMStar (OpenVLM)","family":"mmstar-vlmevalkit","signalToNoise":5.894,"beta":0.864,"sigma":0.147,"communality":0.889,"difficulty":121.9,"nModels":82,"precision":7.2,"precisionMidpoint":4.3,"signalPercentile":52},{"slug":"atlas-exa-200","name":"ATLAS (Exa, 200-task subset)","family":"atlas-exa-200","signalToNoise":5.888,"beta":1.634,"sigma":0.277,"communality":0.7,"difficulty":167.4,"nModels":8,"precision":4.3,"precisionMidpoint":4.3,"signalPercentile":52},{"slug":"docvqa-val","name":"DocVQA (val)","family":"docvqa-val","signalToNoise":5.88,"beta":1.135,"sigma":0.193,"communality":0.123,"difficulty":62.9,"nModels":8,"precision":23.5,"precisionMidpoint":4.3,"signalPercentile":52},{"slug":"spatialbench-long-v1-verified","name":"SpatialBench-Long V0 Verified","family":"spatialbench-long-v1-verified","signalToNoise":5.876,"beta":1.324,"sigma":0.225,"communality":0.233,"difficulty":180.8,"nModels":14,"precision":4.5,"precisionMidpoint":4.3,"signalPercentile":52},{"slug":"global-piqa-nonparallel-loglikelihood-accuracy","name":"Global PIQA - Non-Parallel (Log-Likelihood Accuracy)","family":"global-piqa-nonparallel-loglikelihood-accuracy","signalToNoise":5.867,"beta":1.004,"sigma":0.171,"communality":0.419,"difficulty":153.9,"nModels":20,"precision":7.2,"precisionMidpoint":4.3,"signalPercentile":52},{"slug":"graphwalks-bfs-up-to-128k","name":"GraphWalks BFS: Up to 128K","family":"graphwalks-bfs-up-to-128k","signalToNoise":5.867,"beta":2.099,"sigma":0.358,"communality":0.749,"difficulty":132.5,"nModels":22,"precision":9.9,"precisionMidpoint":4.3,"signalPercentile":52},{"slug":"wildbench-helm-capabilities","name":"WildBench v2 (HELM Capabilities)","family":"wildbench-helm-capabilities","signalToNoise":5.859,"beta":0.61,"sigma":0.104,"communality":0.92,"difficulty":73.1,"nModels":55,"precision":7.4,"precisionMidpoint":4.3,"signalPercentile":52},{"slug":"mmbench-en-test","name":"MMBench (English, test)","family":"mmbench-en-test","signalToNoise":5.842,"beta":0.885,"sigma":0.152,"communality":0.761,"difficulty":72.7,"nModels":8,"precision":12.0,"precisionMidpoint":4.3,"signalPercentile":51},{"slug":"hermes-bench","name":"Hermes Bench","family":"hermes-bench","signalToNoise":5.827,"beta":1.055,"sigma":0.181,"communality":0.631,"difficulty":139.6,"nModels":15,"precision":5.1,"precisionMidpoint":4.3,"signalPercentile":51},{"slug":"cad-eval","name":"CadEval","family":"cad-eval","signalToNoise":5.782,"beta":2.093,"sigma":0.362,"communality":0.786,"difficulty":139.4,"nModels":17,"precision":7.9,"precisionMidpoint":4.4,"signalPercentile":51},{"slug":"big-finance-bench-private-holdout","name":"BigFinanceBench private holdout (final-answer accuracy)","family":"big-finance-bench-private-holdout","signalToNoise":5.74,"beta":1.242,"sigma":0.216,"communality":0.391,"difficulty":166.7,"nModels":14,"precision":4.4,"precisionMidpoint":4.4,"signalPercentile":51},{"slug":"pharmabench-micro1","name":"micro1 PharmaBench","family":"pharmabench-micro1","signalToNoise":5.715,"beta":1.695,"sigma":0.296,"communality":0.092,"difficulty":195.8,"nModels":12,"precision":6.4,"precisionMidpoint":4.4,"signalPercentile":51},{"slug":"brokenarxiv-overall-matharena","name":"BrokenArXiv \u2014 Overall","family":"brokenarxiv-overall-matharena","signalToNoise":5.703,"beta":2.89,"sigma":0.507,"communality":0.514,"difficulty":168.2,"nModels":10,"precision":4.4,"precisionMidpoint":4.4,"signalPercentile":51},{"slug":"agentic-tool-use-enterprise-scale-ai-2","name":"Agentic Tool Use (Enterprise)","family":"agentic-tool-use-enterprise-scale-ai-2","signalToNoise":5.685,"beta":1.324,"sigma":0.233,"communality":0.821,"difficulty":124.9,"nModels":31,"precision":7.6,"precisionMidpoint":4.4,"signalPercentile":51},{"slug":"extended-nyt-connections-940-quadratic-v1","name":"Extended NYT Connections 940 (Quadratic v1)","family":"extended-nyt-connections-940-quadratic-v1","signalToNoise":5.669,"beta":3.739,"sigma":0.66,"communality":0.741,"difficulty":150.0,"nModels":87,"precision":10.0,"precisionMidpoint":4.4,"signalPercentile":50},{"slug":"cred-v0-92","name":"CRED v0.92","family":"cred-v0-92","signalToNoise":5.666,"beta":3.185,"sigma":0.562,"communality":0.582,"difficulty":144.0,"nModels":23,"precision":10.3,"precisionMidpoint":4.4,"signalPercentile":50},{"slug":"forecastbench-baseline-overall-brier-index","name":"ForecastBench Baseline Overall Brier Index","family":"forecastbench-baseline-overall-brier-index","signalToNoise":5.604,"beta":0.567,"sigma":0.101,"communality":0.379,"difficulty":218.6,"nModels":77,"precision":11.8,"precisionMidpoint":4.5,"signalPercentile":50},{"slug":"hil-bench-human-in-loop-benchmark-scale-ai-2","name":"HiL-Bench (Human-in-Loop Benchmark)","family":"hil-bench-human-in-loop-benchmark-scale-ai-2","signalToNoise":5.602,"beta":3.05,"sigma":0.544,"communality":0.556,"difficulty":162.0,"nModels":27,"precision":4.8,"precisionMidpoint":4.5,"signalPercentile":50},{"slug":"buzzbench","name":"BuzzBench","family":"buzzbench","signalToNoise":5.596,"beta":1.129,"sigma":0.202,"communality":0.92,"difficulty":126.6,"nModels":21,"precision":6.6,"precisionMidpoint":4.5,"signalPercentile":50},{"slug":"collie","name":"COLLIE","family":"collie","signalToNoise":5.581,"beta":2.492,"sigma":0.446,"communality":0.802,"difficulty":123.9,"nModels":13,"precision":34.8,"precisionMidpoint":4.5,"signalPercentile":50},{"slug":"secprbench","name":"SecPRBench (recall)","family":"secprbench","signalToNoise":5.565,"beta":2.197,"sigma":0.395,"communality":0.558,"difficulty":152.6,"nModels":8,"precision":5.6,"precisionMidpoint":4.5,"signalPercentile":50},{"slug":"mrcr-v2-8-needle-128k-average","name":"MRCR v2 (8-needle) - 128K average","family":"mrcr-v2-8-needle-128k-average","signalToNoise":5.56,"beta":2.535,"sigma":0.456,"communality":0.793,"difficulty":144.3,"nModels":46,"precision":8.4,"precisionMidpoint":4.5,"signalPercentile":50},{"slug":"macroscopebench","name":"MacroscopeBench","family":"macroscopebench","signalToNoise":5.559,"beta":1.143,"sigma":0.206,"communality":0.725,"difficulty":139.6,"nModels":14,"precision":5.8,"precisionMidpoint":4.5,"signalPercentile":49},{"slug":"snorkel-finance","name":"SnorkelFinance","family":"snorkel-finance","signalToNoise":5.558,"beta":2.179,"sigma":0.392,"communality":0.836,"difficulty":133.1,"nModels":27,"precision":10.4,"precisionMidpoint":4.5,"signalPercentile":49},{"slug":"aa-briefcase-v1-1","name":"AA-Briefcase Elo","family":"aa-briefcase","signalToNoise":5.553,"beta":2.089,"sigma":0.376,"communality":0.866,"difficulty":153.5,"nModels":153,"precision":4.5,"precisionMidpoint":4.5,"signalPercentile":49},{"slug":"mt-bench-gpt-4-turbo","name":"MT-Bench (GPT-4-Turbo judge, corrected references)","family":"mt-bench-gpt-4-turbo","signalToNoise":5.548,"beta":0.902,"sigma":0.163,"communality":0.724,"difficulty":77.7,"nModels":18,"precision":12.0,"precisionMidpoint":4.5,"signalPercentile":49},{"slug":"chess-puzzles","name":"Chess Puzzles","family":"chess-puzzles","signalToNoise":5.545,"beta":2.214,"sigma":0.399,"communality":0.626,"difficulty":167.3,"nModels":73,"precision":4.6,"precisionMidpoint":4.5,"signalPercentile":49},{"slug":"skillsbench-with-skills","name":"SkillsBench with Skills","family":"skillsbench-with-skills","signalToNoise":5.529,"beta":1.668,"sigma":0.302,"communality":0.604,"difficulty":155.8,"nModels":47,"precision":4.9,"precisionMidpoint":4.6,"signalPercentile":49},{"slug":"infographicvqa-val","name":"InfographicVQA (val)","family":"infographicvqa-val","signalToNoise":5.522,"beta":1.034,"sigma":0.187,"communality":0.951,"difficulty":91.7,"nModels":13,"precision":12.1,"precisionMidpoint":4.6,"signalPercentile":49},{"slug":"livecodebench-vals-public","name":"LiveCodeBench (Vals AI public dataset)","family":"livecodebench-vals-public","signalToNoise":5.518,"beta":1.685,"sigma":0.305,"communality":0.857,"difficulty":129.6,"nModels":122,"precision":9.4,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"usamo-2025-matharena","name":"USAMO 2025","family":"usamo-2025-matharena","signalToNoise":5.514,"beta":2.766,"sigma":0.502,"communality":0.449,"difficulty":162.5,"nModels":10,"precision":4.8,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"tau-bench-retail","name":"\u03c4-bench (Retail)","family":"tau-bench-retail","signalToNoise":5.471,"beta":1.805,"sigma":0.33,"communality":0.726,"difficulty":129.5,"nModels":40,"precision":9.5,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"snorkel-manufacturing","name":"SnorkelManufacturing","family":"snorkel-manufacturing","signalToNoise":5.452,"beta":1.645,"sigma":0.302,"communality":0.047,"difficulty":195.2,"nModels":13,"precision":6.5,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"long-horizon-terminal-bench-2-0","name":"Long-Horizon Terminal-Bench 2.0 (mean reward)","family":"long-horizon-terminal-bench-2-0","signalToNoise":5.451,"beta":1.332,"sigma":0.244,"communality":0.599,"difficulty":156.5,"nModels":12,"precision":4.8,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"scicode-1-0-1-artificial-analysis","name":"SciCode 1.0.1 (Artificial Analysis)","family":"scicode-1-0-1-artificial-analysis","signalToNoise":5.45,"beta":0.957,"sigma":0.176,"communality":0.851,"difficulty":156.4,"nModels":150,"precision":4.7,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"learn2play-bench","name":"Learn2Play Bench","family":"learn2play-bench","signalToNoise":5.438,"beta":1.715,"sigma":0.315,"communality":0.725,"difficulty":150.3,"nModels":9,"precision":5.5,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"swe-bench-verified","name":"SWE-bench Verified","family":"swe-bench-verified","signalToNoise":5.422,"beta":1.966,"sigma":0.363,"communality":0.778,"difficulty":136.9,"nModels":169,"precision":8.3,"precisionMidpoint":4.6,"signalPercentile":48},{"slug":"swe-atlas-codebase-qna-scale-ai-2","name":"SWE Atlas - Codebase QnA","family":"swe-atlas-codebase-qna-scale-ai-2","signalToNoise":5.411,"beta":2.163,"sigma":0.4,"communality":0.551,"difficulty":163.7,"nModels":53,"precision":4.7,"precisionMidpoint":4.7,"signalPercentile":47},{"slug":"ide-bench","name":"IDE-Bench","family":"ide-bench","signalToNoise":5.407,"beta":3.758,"sigma":0.695,"communality":0.724,"difficulty":144.4,"nModels":14,"precision":13.6,"precisionMidpoint":4.7,"signalPercentile":47},{"slug":"judgemark-v2-1","name":"Judgemark v2.1","family":"judgemark-v2-1","signalToNoise":5.381,"beta":1.827,"sigma":0.34,"communality":0.756,"difficulty":129.6,"nModels":45,"precision":9.7,"precisionMidpoint":4.7,"signalPercentile":47},{"slug":"real-swe","name":"Real-SWE","family":"real-swe","signalToNoise":5.375,"beta":1.631,"sigma":0.303,"communality":0.368,"difficulty":174.1,"nModels":8,"precision":4.8,"precisionMidpoint":4.7,"signalPercentile":47},{"slug":"cnmo-2024","name":"CNMO 2024","family":"cnmo-2024","signalToNoise":5.369,"beta":2.773,"sigma":0.516,"communality":0.868,"difficulty":142.6,"nModels":11,"precision":22.4,"precisionMidpoint":4.7,"signalPercentile":47},{"slug":"vita-bench","name":"VitaBench","family":"vita-bench","signalToNoise":5.35,"beta":1.638,"sigma":0.306,"communality":0.676,"difficulty":155.7,"nModels":25,"precision":5.1,"precisionMidpoint":4.7,"signalPercentile":47},{"slug":"aime-2025","name":"AIME 2025","family":"aime-2025","signalToNoise":5.335,"beta":2.641,"sigma":0.495,"communality":0.63,"difficulty":131.6,"nModels":185,"precision":21.6,"precisionMidpoint":4.7,"signalPercentile":47},{"slug":"medqa-vals-demographic-bias","name":"MedQA Demographic Bias (Vals)","family":"medqa-vals-demographic-bias","signalToNoise":5.325,"beta":1.624,"sigma":0.305,"communality":0.79,"difficulty":113.2,"nModels":80,"precision":17.2,"precisionMidpoint":4.7,"signalPercentile":46},{"slug":"dayjob-law","name":"DAYJOB: Law","family":"dayjob-law","signalToNoise":5.309,"beta":2.547,"sigma":0.48,"communality":0.443,"difficulty":181.0,"nModels":13,"precision":5.7,"precisionMidpoint":4.7,"signalPercentile":46},{"slug":"diligence-bench","name":"DiligenceBench","family":"diligence-bench","signalToNoise":5.306,"beta":1.269,"sigma":0.239,"communality":0.488,"difficulty":170.8,"nModels":16,"precision":4.8,"precisionMidpoint":4.7,"signalPercentile":46},{"slug":"mmmlu","name":"MMMLU","family":"mmmlu","signalToNoise":5.297,"beta":1.375,"sigma":0.26,"communality":0.798,"difficulty":118.4,"nModels":75,"precision":11.2,"precisionMidpoint":4.8,"signalPercentile":46},{"slug":"scbench-v2","name":"ScBench V1 Clean Reruns","family":"scbench-v2","signalToNoise":5.287,"beta":1.224,"sigma":0.232,"communality":0.379,"difficulty":157.2,"nModels":24,"precision":4.9,"precisionMidpoint":4.8,"signalPercentile":46},{"slug":"otis-mock-aime-2024-2025","name":"OTIS Mock AIME 2024-2025","family":"otis-mock-aime-2024-2025","signalToNoise":5.254,"beta":2.7,"sigma":0.514,"communality":0.707,"difficulty":134.5,"nModels":131,"precision":16.7,"precisionMidpoint":4.8,"signalPercentile":46},{"slug":"livecodebench-v6-reported","name":"LiveCodeBench","family":"livecodebench","signalToNoise":5.231,"beta":2.053,"sigma":0.393,"communality":0.843,"difficulty":131.8,"nModels":75,"precision":11.7,"precisionMidpoint":4.8,"signalPercentile":46},{"slug":"public-benefits-bench","name":"Public Benefits Bench v1.1","family":"public-benefits-bench","signalToNoise":5.229,"beta":0.929,"sigma":0.178,"communality":0.709,"difficulty":143.9,"nModels":47,"precision":5.3,"precisionMidpoint":4.8,"signalPercentile":46},{"slug":"mlcr-aa-2026-10","name":"Medical Long Context Reasoning (MLCR-AA, October 2026 methodology)","family":"mlcr-aa-2026-10","signalToNoise":5.199,"beta":1.815,"sigma":0.349,"communality":0.6,"difficulty":181.3,"nModels":99,"precision":5.4,"precisionMidpoint":4.8,"signalPercentile":45},{"slug":"fictionlivebench-60k-tokens","name":"Fiction.LiveBench - 60K tokens","family":"fictionlivebench-60k-tokens","signalToNoise":5.194,"beta":3.026,"sigma":0.583,"communality":0.369,"difficulty":136.7,"nModels":47,"precision":16.4,"precisionMidpoint":4.8,"signalPercentile":45},{"slug":"arc-challenge","name":"ARC AI2","family":"arc-ai2","signalToNoise":5.14,"beta":1.943,"sigma":0.378,"communality":0.859,"difficulty":93.6,"nModels":101,"precision":50.8,"precisionMidpoint":4.9,"signalPercentile":45},{"slug":"spatialbench-v1-verified","name":"SpatialBench V0 Verified","family":"spatialbench-v1-verified","signalToNoise":5.129,"beta":0.959,"sigma":0.187,"communality":0.183,"difficulty":137.3,"nModels":16,"precision":5.8,"precisionMidpoint":4.9,"signalPercentile":45},{"slug":"astabench-2026-ds1000","name":"DS-1000 (AstaBench 2026)","family":"astabench-2026-ds1000","signalToNoise":5.127,"beta":1.16,"sigma":0.226,"communality":0.848,"difficulty":124.5,"nModels":13,"precision":7.6,"precisionMidpoint":4.9,"signalPercentile":45},{"slug":"hlab","name":"Harvey's Legal Agent Benchmark","family":"hlab","signalToNoise":5.102,"beta":2.196,"sigma":0.43,"communality":0.276,"difficulty":188.6,"nModels":74,"precision":7.2,"precisionMidpoint":4.9,"signalPercentile":45},{"slug":"mlcr-aa","name":"Medical Long Context Reasoning (MLCR-AA)","family":"mlcr-aa","signalToNoise":5.095,"beta":2.698,"sigma":0.53,"communality":0.568,"difficulty":168.9,"nModels":70,"precision":5.0,"precisionMidpoint":4.9,"signalPercentile":45},{"slug":"babaisbench-intro","name":"BabaIsBench Stage 0: The Intro","family":"babaisbench-intro","signalToNoise":5.081,"beta":4.162,"sigma":0.819,"communality":0.201,"difficulty":150.4,"nModels":30,"precision":11.5,"precisionMidpoint":5.0,"signalPercentile":44},{"slug":"aa-lcr-1-1-artificial-analysis","name":"AA-LCR v1.1","family":"aa-lcr-1-1-artificial-analysis","signalToNoise":5.077,"beta":1.971,"sigma":0.388,"communality":0.452,"difficulty":140.4,"nModels":390,"precision":8.3,"precisionMidpoint":5.0,"signalPercentile":44},{"slug":"snorkel-graph","name":"SnorkelGraph","family":"snorkel-graph","signalToNoise":5.053,"beta":1.828,"sigma":0.362,"communality":0.845,"difficulty":135.4,"nModels":30,"precision":9.0,"precisionMidpoint":5.0,"signalPercentile":44},{"slug":"gamedevbench","name":"GameDevBench","family":"gamedevbench","signalToNoise":5.049,"beta":2.254,"sigma":0.446,"communality":0.634,"difficulty":158.2,"nModels":18,"precision":5.5,"precisionMidpoint":5.0,"signalPercentile":44},{"slug":"eq-bench-4","name":"EQ-Bench 4","family":"eq-bench-4","signalToNoise":5.044,"beta":2.839,"sigma":0.563,"communality":0.709,"difficulty":154.3,"nModels":31,"precision":5.0,"precisionMidpoint":5.0,"signalPercentile":44},{"slug":"llm-persuasion-average-strength","name":"LLM Persuasion Average Strength","family":"llm-persuasion-average-strength","signalToNoise":5.012,"beta":0.485,"sigma":0.097,"communality":0.721,"difficulty":128.5,"nModels":15,"precision":5.4,"precisionMidpoint":5.0,"signalPercentile":44},{"slug":"poker_agent","name":"Poker Agent","family":"poker_agent","signalToNoise":4.992,"beta":2.981,"sigma":0.597,"communality":0.686,"difficulty":145.7,"nModels":17,"precision":5.0,"precisionMidpoint":5.0,"signalPercentile":44},{"slug":"tau2-bench-average","name":"\u03c4\u00b2-bench (domain average)","family":"tau2-bench-average","signalToNoise":4.981,"beta":1.812,"sigma":0.364,"communality":0.831,"difficulty":128.7,"nModels":21,"precision":10.9,"precisionMidpoint":5.1,"signalPercentile":44},{"slug":"videomme","name":"Video-MME","family":"videomme","signalToNoise":4.978,"beta":1.011,"sigma":0.203,"communality":0.802,"difficulty":121.8,"nModels":62,"precision":8.9,"precisionMidpoint":5.1,"signalPercentile":43},{"slug":"video-mmmu","name":"Video-MMMU","family":"video-mmmu","signalToNoise":4.967,"beta":1.256,"sigma":0.253,"communality":0.843,"difficulty":120.1,"nModels":70,"precision":9.6,"precisionMidpoint":5.1,"signalPercentile":43},{"slug":"cmrc-2018","name":"CMRC 2018","family":"cmrc-2018","signalToNoise":4.966,"beta":0.807,"sigma":0.163,"communality":0.206,"difficulty":80.7,"nModels":8,"precision":11.0,"precisionMidpoint":5.1,"signalPercentile":43},{"slug":"mgsm","name":"MGSM","family":"mgsm","signalToNoise":4.964,"beta":1.561,"sigma":0.314,"communality":0.845,"difficulty":101.5,"nModels":116,"precision":20.0,"precisionMidpoint":5.1,"signalPercentile":43},{"slug":"hmmt-feb-2025-matharena","name":"HMMT Feb 2025","family":"hmmt-feb-2025-matharena","signalToNoise":4.936,"beta":3.332,"sigma":0.675,"communality":0.671,"difficulty":136.1,"nModels":78,"precision":27.9,"precisionMidpoint":5.1,"signalPercentile":43},{"slug":"tau2-bench-airline-50","name":"\u03c4\u00b2-bench Airline - 50 tasks","family":"tau2-bench-airline-50","signalToNoise":4.92,"beta":1.747,"sigma":0.355,"communality":0.761,"difficulty":137.5,"nModels":58,"precision":8.3,"precisionMidpoint":5.1,"signalPercentile":43},{"slug":"api-bank-meta-corrected","name":"API-Bank (Meta corrected protocol)","family":"api-bank-meta-corrected","signalToNoise":4.88,"beta":1.939,"sigma":0.397,"communality":0.735,"difficulty":98.2,"nModels":9,"precision":38.7,"precisionMidpoint":5.2,"signalPercentile":43},{"slug":"vials","name":"VIALS","family":"vials","signalToNoise":4.878,"beta":1.823,"sigma":0.374,"communality":0.578,"difficulty":175.2,"nModels":20,"precision":5.3,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"livecodebench-pro","name":"LiveCodeBench Pro","family":"livecodebench-pro","signalToNoise":4.863,"beta":2.407,"sigma":0.495,"communality":0.783,"difficulty":141.4,"nModels":34,"precision":5.2,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"treasuryqa-bench","name":"TreasuryQA Bench","family":"treasuryqa-bench","signalToNoise":4.86,"beta":2.416,"sigma":0.497,"communality":0.377,"difficulty":168.7,"nModels":10,"precision":5.2,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"prinzbench","name":"PrinzBench","family":"prinzbench","signalToNoise":4.854,"beta":2.867,"sigma":0.591,"communality":0.577,"difficulty":159.4,"nModels":26,"precision":6.1,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"teacharena","name":"TeachArena","family":"teacharena","signalToNoise":4.833,"beta":0.697,"sigma":0.144,"communality":0.589,"difficulty":115.1,"nModels":15,"precision":6.7,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"mmbench-v1-1-dev-en","name":"MMBench v1.1 (dev, English)","family":"mmbench-v1-1-dev-en","signalToNoise":4.827,"beta":0.921,"sigma":0.191,"communality":0.752,"difficulty":86.3,"nModels":46,"precision":13.2,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"legalreasonbench","name":"LegalReasonBench","family":"legalreasonbench","signalToNoise":4.824,"beta":1.131,"sigma":0.234,"communality":0.299,"difficulty":172.5,"nModels":8,"precision":5.2,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"buyout-game-bradley-terry","name":"Buyout Game Bradley-Terry Rating","family":"buyout-game-bradley-terry","signalToNoise":4.821,"beta":2.872,"sigma":0.596,"communality":0.669,"difficulty":149.1,"nModels":32,"precision":5.2,"precisionMidpoint":5.2,"signalPercentile":42},{"slug":"public-benefits-bench-both","name":"Public Benefits Bench v1 - Both","family":"public-benefits-bench-both","signalToNoise":4.813,"beta":0.869,"sigma":0.181,"communality":0.666,"difficulty":145.7,"nModels":13,"precision":5.6,"precisionMidpoint":5.2,"signalPercentile":41},{"slug":"hemingway-bench-surge-ai-2","name":"Hemingway-bench","family":"hemingway-bench-surge-ai-2","signalToNoise":4.805,"beta":2.631,"sigma":0.548,"communality":0.729,"difficulty":154.2,"nModels":40,"precision":5.2,"precisionMidpoint":5.2,"signalPercentile":41},{"slug":"we-math","name":"We-Math","family":"we-math","signalToNoise":4.796,"beta":1.916,"sigma":0.399,"communality":0.664,"difficulty":129.8,"nModels":10,"precision":12.0,"precisionMidpoint":5.2,"signalPercentile":41},{"slug":"livecodebench-artificial-analysis","name":"LiveCodeBench (Artificial Analysis)","family":"livecodebench-artificial-analysis","signalToNoise":4.789,"beta":1.982,"sigma":0.414,"communality":0.873,"difficulty":132.9,"nModels":265,"precision":12.0,"precisionMidpoint":5.3,"signalPercentile":41},{"slug":"swe-bench-verified-mercor-extended","name":"SWE-bench Verified Extended (Mercor held-out)","family":"swe-bench-verified-mercor-extended","signalToNoise":4.784,"beta":2.927,"sigma":0.612,"communality":0.523,"difficulty":158.6,"nModels":41,"precision":5.9,"precisionMidpoint":5.3,"signalPercentile":41},{"slug":"coding-scale-ai-2","name":"Coding","family":"coding-scale-ai-2","signalToNoise":4.778,"beta":2.156,"sigma":0.451,"communality":0.839,"difficulty":126.2,"nModels":28,"precision":5.3,"precisionMidpoint":5.3,"signalPercentile":41},{"slug":"snorkel-finance-2","name":"SnorkelFinance 2.0","family":"snorkel-finance-2","signalToNoise":4.761,"beta":1.62,"sigma":0.34,"communality":0.002,"difficulty":193.7,"nModels":13,"precision":7.1,"precisionMidpoint":5.3,"signalPercentile":41},{"slug":"scbench-long-v2","name":"ScBench-Long V1 Clean Reruns","family":"scbench-long-v2","signalToNoise":4.758,"beta":1.626,"sigma":0.342,"communality":0.215,"difficulty":187.4,"nModels":24,"precision":6.3,"precisionMidpoint":5.3,"signalPercentile":40},{"slug":"ebr-bench-v1","name":"EBR-bench v1","family":"ebr-bench-v1","signalToNoise":4.757,"beta":1.813,"sigma":0.381,"communality":0.385,"difficulty":185.5,"nModels":12,"precision":6.2,"precisionMidpoint":5.3,"signalPercentile":40},{"slug":"graphwalks-parents-up-to-128k","name":"GraphWalks Parents: Up to 128K","family":"graphwalks-parents-up-to-128k","signalToNoise":4.753,"beta":2.022,"sigma":0.425,"communality":0.8,"difficulty":137.5,"nModels":18,"precision":9.8,"precisionMidpoint":5.3,"signalPercentile":40},{"slug":"medxpertqa-mm-200","name":"MedXpertQA MM (200-question set)","family":"medxpertqa-mm-200","signalToNoise":4.745,"beta":1.378,"sigma":0.29,"communality":0.516,"difficulty":147.5,"nModels":39,"precision":7.4,"precisionMidpoint":5.3,"signalPercentile":40},{"slug":"arc-challenge-openllm-v1-harness","name":"ARC-Challenge (Open LLM Leaderboard v1)","family":"arc-challenge-openllm-v1-harness","signalToNoise":4.728,"beta":1.019,"sigma":0.216,"communality":0.833,"difficulty":103.3,"nModels":118,"precision":12.4,"precisionMidpoint":5.3,"signalPercentile":40},{"slug":"globebench","name":"GlobeBench","family":"globebench","signalToNoise":4.717,"beta":1.958,"sigma":0.415,"communality":0.359,"difficulty":181.1,"nModels":17,"precision":6.0,"precisionMidpoint":5.3,"signalPercentile":40},{"slug":"chess-puzzles-v2","name":"Chess Puzzles (task version 2.0)","family":"chess-puzzles-v2","signalToNoise":4.71,"beta":2.009,"sigma":0.426,"communality":0.639,"difficulty":164.8,"nModels":81,"precision":5.5,"precisionMidpoint":5.3,"signalPercentile":40},{"slug":"global-piqa-parallel-exact-match-strict","name":"Global PIQA - Parallel (Strict Exact Match)","family":"global-piqa-parallel-exact-match-strict","signalToNoise":4.699,"beta":1.454,"sigma":0.31,"communality":0.735,"difficulty":135.7,"nModels":53,"precision":9.3,"precisionMidpoint":5.4,"signalPercentile":40},{"slug":"mathverse-mini","name":"MathVerse (testmini)","family":"mathverse-mini","signalToNoise":4.696,"beta":1.232,"sigma":0.262,"communality":0.865,"difficulty":118.9,"nModels":24,"precision":11.0,"precisionMidpoint":5.4,"signalPercentile":39},{"slug":"xbench-scienceqa","name":"xbench-ScienceQA","family":"xbench-scienceqa","signalToNoise":4.685,"beta":0.561,"sigma":0.12,"communality":0.435,"difficulty":117.5,"nModels":27,"precision":6.3,"precisionMidpoint":5.4,"signalPercentile":39},{"slug":"mle-bench-partial-30-average-position","name":"MLE-Bench Partial 30 - Average Position Score","family":"mle-bench-partial-30-average-position","signalToNoise":4.649,"beta":1.722,"sigma":0.37,"communality":0.566,"difficulty":157.4,"nModels":8,"precision":5.8,"precisionMidpoint":5.4,"signalPercentile":39},{"slug":"tau-bench-airline","name":"\u03c4-bench (Airline)","family":"tau-bench-airline","signalToNoise":4.644,"beta":1.473,"sigma":0.317,"communality":0.675,"difficulty":141.7,"nModels":40,"precision":7.1,"precisionMidpoint":5.4,"signalPercentile":39},{"slug":"snorkel-wordle","name":"SnorkelWordle","family":"snorkel-wordle","signalToNoise":4.63,"beta":2.635,"sigma":0.569,"communality":0.555,"difficulty":128.8,"nModels":20,"precision":21.8,"precisionMidpoint":5.4,"signalPercentile":39},{"slug":"voratiq-coding-agent-leaderboard","name":"Voratiq Coding Agent Leaderboard","family":"voratiq-coding-agent-leaderboard","signalToNoise":4.616,"beta":3.144,"sigma":0.681,"communality":0.548,"difficulty":156.9,"nModels":19,"precision":5.5,"precisionMidpoint":5.5,"signalPercentile":39},{"slug":"cyberarena","name":"CyberArena","family":"cyberarena","signalToNoise":4.612,"beta":1.206,"sigma":0.262,"communality":0.308,"difficulty":144.0,"nModels":11,"precision":6.4,"precisionMidpoint":5.5,"signalPercentile":39},{"slug":"design-arena-overall-code-categories","name":"Design Arena Overall Code Categories","family":"design-arena-overall-code-categories","signalToNoise":4.608,"beta":2.03,"sigma":0.441,"communality":0.816,"difficulty":147.9,"nModels":150,"precision":5.5,"precisionMidpoint":5.5,"signalPercentile":38},{"slug":"forecastbench-tournament-overall-brier-index","name":"ForecastBench Tournament Overall Brier Index","family":"forecastbench-tournament-overall-brier-index","signalToNoise":4.584,"beta":0.415,"sigma":0.091,"communality":0.55,"difficulty":209.7,"nModels":88,"precision":11.5,"precisionMidpoint":5.5,"signalPercentile":38},{"slug":"math-lvl-5-openllm-harness","name":"MATH Level 5 (Open LLM Leaderboard v2)","family":"math-lvl-5-openllm-harness","signalToNoise":4.577,"beta":1.873,"sigma":0.409,"communality":0.681,"difficulty":130.7,"nModels":121,"precision":11.9,"precisionMidpoint":5.5,"signalPercentile":38},{"slug":"terminal-bench-1-0","name":"Terminal-Bench 1.0","family":"terminal-bench-1-0","signalToNoise":4.558,"beta":1.992,"sigma":0.437,"communality":0.324,"difficulty":151.7,"nModels":15,"precision":6.6,"precisionMidpoint":5.5,"signalPercentile":38},{"slug":"alpacaeval-2-lc-winrate","name":"AlpacaEval 2.0 (length-controlled win rate)","family":"alpacaeval-2-lc-winrate","signalToNoise":4.546,"beta":1.472,"sigma":0.324,"communality":0.864,"difficulty":126.4,"nModels":61,"precision":10.2,"precisionMidpoint":5.5,"signalPercentile":38},{"slug":"lmarena-vision-arena","name":"Vision Arena Overall","family":"lmarena-vision-arena","signalToNoise":4.534,"beta":1.534,"sigma":0.338,"communality":0.895,"difficulty":145.2,"nModels":103,"precision":5.6,"precisionMidpoint":5.6,"signalPercentile":38},{"slug":"crm-agent-bench","name":"CRM-Agent-Bench","family":"crm-agent-bench","signalToNoise":4.527,"beta":0.679,"sigma":0.15,"communality":0.785,"difficulty":101.0,"nModels":14,"precision":7.9,"precisionMidpoint":5.6,"signalPercentile":38},{"slug":"mmvu","name":"MMVU","family":"mmvu","signalToNoise":4.517,"beta":0.851,"sigma":0.188,"communality":0.879,"difficulty":118.3,"nModels":40,"precision":7.7,"precisionMidpoint":5.6,"signalPercentile":38},{"slug":"itbench-aa-artificial-analysis","name":"ITBench-AA","family":"itbench-aa-artificial-analysis","signalToNoise":4.501,"beta":1.826,"sigma":0.406,"communality":0.606,"difficulty":163.0,"nModels":42,"precision":5.7,"precisionMidpoint":5.6,"signalPercentile":37},{"slug":"mmmu-val-vlmevalkit","name":"MMMU Validation (OpenVLM)","family":"mmmu-val-vlmevalkit","signalToNoise":4.492,"beta":0.754,"sigma":0.168,"communality":0.89,"difficulty":108.4,"nModels":58,"precision":8.0,"precisionMidpoint":5.6,"signalPercentile":37},{"slug":"blenderworld","name":"BlenderWorld","family":"blenderworld","signalToNoise":4.48,"beta":1.804,"sigma":0.403,"communality":0.434,"difficulty":154.7,"nModels":10,"precision":6.3,"precisionMidpoint":5.6,"signalPercentile":37},{"slug":"snorkel-underwrite-2","name":"SnorkelUnderwrite 2.0","family":"snorkel-underwrite-2","signalToNoise":4.474,"beta":1.028,"sigma":0.23,"communality":0.0,"difficulty":196.2,"nModels":13,"precision":6.6,"precisionMidpoint":5.6,"signalPercentile":37},{"slug":"superchem","name":"SUPERChem","family":"superchem","signalToNoise":4.474,"beta":1.826,"sigma":0.408,"communality":0.462,"difficulty":154.1,"nModels":29,"precision":6.5,"precisionMidpoint":5.6,"signalPercentile":37},{"slug":"medxpertqa-mm-mercor-extended","name":"MedXpertQA MM Extended (Mercor held-out)","family":"medxpertqa-mm-mercor-extended","signalToNoise":4.471,"beta":1.036,"sigma":0.232,"communality":0.581,"difficulty":156.9,"nModels":39,"precision":5.9,"precisionMidpoint":5.6,"signalPercentile":37},{"slug":"surface-evolver-bench","name":"Surface Evolver Bench","family":"surface-evolver-bench","signalToNoise":4.463,"beta":2.758,"sigma":0.618,"communality":0.596,"difficulty":151.3,"nModels":26,"precision":8.2,"precisionMidpoint":5.6,"signalPercentile":37},{"slug":"cadqueryeval-corrected-geometry","name":"CadQueryEval (Corrected Geometry)","family":"cadqueryeval-corrected-geometry","signalToNoise":4.456,"beta":2.485,"sigma":0.558,"communality":0.371,"difficulty":150.2,"nModels":86,"precision":8.0,"precisionMidpoint":5.6,"signalPercentile":36},{"slug":"piqa","name":"PIQA","family":"piqa","signalToNoise":4.399,"beta":0.387,"sigma":0.088,"communality":0.856,"difficulty":66.2,"nModels":81,"precision":11.3,"precisionMidpoint":5.7,"signalPercentile":36},{"slug":"winogrande-openllm-v1-harness","name":"WinoGrande (Open LLM Leaderboard v1)","family":"winogrande-openllm-v1-harness","signalToNoise":4.368,"beta":0.963,"sigma":0.22,"communality":0.699,"difficulty":98.5,"nModels":115,"precision":16.9,"precisionMidpoint":5.8,"signalPercentile":36},{"slug":"mystery-game-puzzles","name":"Mystery Game Puzzles","family":"mystery-game-puzzles","signalToNoise":4.351,"beta":2.104,"sigma":0.484,"communality":0.433,"difficulty":167.6,"nModels":74,"precision":5.8,"precisionMidpoint":5.8,"signalPercentile":36},{"slug":"veracode-genai-code-security-fall-2026","name":"Veracode GenAI Code Security (Fall 2026)","family":"veracode-genai-code-security-fall-2026","signalToNoise":4.341,"beta":0.841,"sigma":0.194,"communality":0.732,"difficulty":141.2,"nModels":16,"precision":6.4,"precisionMidpoint":5.8,"signalPercentile":36},{"slug":"biomysterybench-human-difficult","name":"BioMysteryBench - Human Difficult","family":"biomysterybench-human-difficult","signalToNoise":4.334,"beta":1.032,"sigma":0.238,"communality":0.227,"difficulty":166.8,"nModels":9,"precision":5.8,"precisionMidpoint":5.8,"signalPercentile":36},{"slug":"agentic-tool-use-chat-scale-ai-2","name":"Agentic Tool Use (Chat)","family":"agentic-tool-use-chat-scale-ai-2","signalToNoise":4.324,"beta":1.318,"sigma":0.305,"communality":0.685,"difficulty":130.1,"nModels":30,"precision":9.0,"precisionMidpoint":5.8,"signalPercentile":36},{"slug":"clockbench-accuracy","name":"ClockBench Accuracy","family":"clockbench-accuracy","signalToNoise":4.306,"beta":2.397,"sigma":0.557,"communality":0.704,"difficulty":164.5,"nModels":38,"precision":6.0,"precisionMidpoint":5.8,"signalPercentile":36},{"slug":"aml-bench","name":"AML Bench","family":"aml-bench","signalToNoise":4.305,"beta":0.707,"sigma":0.164,"communality":0.087,"difficulty":135.8,"nModels":11,"precision":6.5,"precisionMidpoint":5.8,"signalPercentile":35},{"slug":"charxiv-reasoning","name":"CharXiv Reasoning","family":"charxiv-reasoning","signalToNoise":4.297,"beta":1.197,"sigma":0.279,"communality":0.873,"difficulty":128.2,"nModels":129,"precision":8.9,"precisionMidpoint":5.9,"signalPercentile":35},{"slug":"mcp-atlas","name":"MCP Atlas (original protocol)","family":"mcp-atlas","signalToNoise":4.285,"beta":2.528,"sigma":0.59,"communality":0.339,"difficulty":147.8,"nModels":27,"precision":9.0,"precisionMidpoint":5.9,"signalPercentile":35},{"slug":"ocrbench-v2-en","name":"OCRBench v2","family":"ocrbench-v2","signalToNoise":4.283,"beta":0.606,"sigma":0.142,"communality":0.9,"difficulty":117.9,"nModels":8,"precision":7.1,"precisionMidpoint":5.9,"signalPercentile":35},{"slug":"multichallenge-scale-ai-2","name":"MultiChallenge","family":"multichallenge-scale-ai-2","signalToNoise":4.274,"beta":1.163,"sigma":0.272,"communality":0.828,"difficulty":141.2,"nModels":66,"precision":7.1,"precisionMidpoint":5.9,"signalPercentile":35},{"slug":"stembench120-1-0","name":"STEMBench120 v1.0.0 (closed-book)","family":"stembench120-1-0","signalToNoise":4.244,"beta":0.759,"sigma":0.179,"communality":0.074,"difficulty":141.0,"nModels":8,"precision":6.4,"precisionMidpoint":5.9,"signalPercentile":35},{"slug":"lmarena-text-arena-elo","name":"LMArena Text Arena Elo","family":"lmarena-text-arena-elo","signalToNoise":4.224,"beta":1.147,"sigma":0.272,"communality":0.932,"difficulty":136.0,"nModels":240,"precision":6.0,"precisionMidpoint":6.0,"signalPercentile":35},{"slug":"turing-vlmbench","name":"Turing VLMBench","family":"turing-vlmbench","signalToNoise":4.211,"beta":1.205,"sigma":0.286,"communality":0.703,"difficulty":159.9,"nModels":17,"precision":6.1,"precisionMidpoint":6.0,"signalPercentile":34},{"slug":"vibevideobench","name":"VibeVideoBench (overall quality, no video = 0)","family":"vibevideobench","signalToNoise":4.163,"beta":1.495,"sigma":0.359,"communality":0.402,"difficulty":145.4,"nModels":8,"precision":7.5,"precisionMidpoint":6.0,"signalPercentile":34},{"slug":"arena-ai-document","name":"Document Arena","family":"arena-ai-document","signalToNoise":4.156,"beta":2.912,"sigma":0.701,"communality":0.511,"difficulty":156.0,"nModels":35,"precision":6.1,"precisionMidpoint":6.1,"signalPercentile":34},{"slug":"activevision","name":"ActiveVision","family":"activevision","signalToNoise":4.153,"beta":3.111,"sigma":0.749,"communality":0.406,"difficulty":173.0,"nModels":13,"precision":6.3,"precisionMidpoint":6.1,"signalPercentile":34},{"slug":"commonsenseqa","name":"CommonSenseQA","family":"commonsenseqa","signalToNoise":4.145,"beta":1.5,"sigma":0.362,"communality":0.825,"difficulty":89.6,"nModels":18,"precision":35.9,"precisionMidpoint":6.1,"signalPercentile":34},{"slug":"medmcqa","name":"MedMCQA","family":"medmcqa","signalToNoise":4.123,"beta":0.973,"sigma":0.236,"communality":0.702,"difficulty":125.6,"nModels":58,"precision":9.8,"precisionMidpoint":6.1,"signalPercentile":34},{"slug":"terminal-bench-2-1-mercor-extended","name":"Terminal-Bench 2.1 Extended (Mercor held-out)","family":"terminal-bench-2-1-mercor-extended","signalToNoise":4.122,"beta":1.426,"sigma":0.346,"communality":0.494,"difficulty":174.3,"nModels":41,"precision":6.2,"precisionMidpoint":6.1,"signalPercentile":34},{"slug":"pointerbench","name":"Pointerbench","family":"pointerbench","signalToNoise":4.111,"beta":3.676,"sigma":0.894,"communality":0.304,"difficulty":163.4,"nModels":29,"precision":6.5,"precisionMidpoint":6.1,"signalPercentile":34},{"slug":"gsm8k","name":"GSM8K","family":"gsm8k","signalToNoise":4.107,"beta":1.81,"sigma":0.441,"communality":0.858,"difficulty":91.6,"nModels":210,"precision":50.2,"precisionMidpoint":6.1,"signalPercentile":33},{"slug":"samaya-frontierfinance-v3-1","name":"FrontierFinance v3.1","family":"samaya-frontierfinance-v3-1","signalToNoise":4.097,"beta":0.946,"sigma":0.231,"communality":0.381,"difficulty":165.5,"nModels":13,"precision":6.1,"precisionMidpoint":6.1,"signalPercentile":33},{"slug":"androidworld-reported","name":"AndroidWorld - release-reported mobile use","family":"androidworld-reported","signalToNoise":4.092,"beta":1.254,"sigma":0.306,"communality":0.79,"difficulty":134.2,"nModels":33,"precision":8.6,"precisionMidpoint":6.2,"signalPercentile":33},{"slug":"bazaar-trueskill-mu","name":"BAZAAR TrueSkill Mu","family":"bazaar-trueskill-mu","signalToNoise":4.061,"beta":2.289,"sigma":0.564,"communality":0.712,"difficulty":134.8,"nModels":22,"precision":6.2,"precisionMidpoint":6.2,"signalPercentile":33},{"slug":"spatialbench-long-v2","name":"SpatialBench-Long V1 Clean Reruns","family":"spatialbench-long-v2","signalToNoise":4.041,"beta":1.051,"sigma":0.26,"communality":0.148,"difficulty":186.2,"nModels":24,"precision":6.7,"precisionMidpoint":6.2,"signalPercentile":33},{"slug":"dualbench100","name":"DualEntry Accounting AI Benchmark","family":"dualbench100","signalToNoise":4.023,"beta":1.019,"sigma":0.253,"communality":0.067,"difficulty":120.9,"nModels":22,"precision":9.4,"precisionMidpoint":6.3,"signalPercentile":33},{"slug":"apex-v1-extended","name":"APEX-1 (Mercor, mean score)","family":"apex-v1-extended","signalToNoise":4.021,"beta":0.888,"sigma":0.221,"communality":0.507,"difficulty":141.7,"nModels":65,"precision":7.0,"precisionMidpoint":6.3,"signalPercentile":33},{"slug":"global-piqa-nonparallel-exact-match-strict","name":"Global PIQA - Non-Parallel (Strict Exact Match)","family":"global-piqa-nonparallel-exact-match-strict","signalToNoise":4.015,"beta":1.211,"sigma":0.302,"communality":0.573,"difficulty":132.6,"nModels":54,"precision":12.8,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"bbh-openllm-harness","name":"BBH (Open LLM Leaderboard v2)","family":"bbh-openllm-harness","signalToNoise":4.008,"beta":0.713,"sigma":0.178,"communality":0.861,"difficulty":101.7,"nModels":121,"precision":9.3,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"pact-negotiation-bilateral-rating","name":"PACT Negotiation Bilateral Rating","family":"pact-negotiation-bilateral-rating","signalToNoise":4.005,"beta":2.133,"sigma":0.533,"communality":0.756,"difficulty":148.4,"nModels":25,"precision":6.3,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"snorkel-finance-reasoning","name":"Snorkel Finance Reasoning","family":"snorkel-finance-reasoning","signalToNoise":3.998,"beta":1.36,"sigma":0.34,"communality":0.674,"difficulty":151.0,"nModels":25,"precision":7.0,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"zerobench-subquestions-pass-1-zerobench","name":"ZeroBench (Subquestions, pass@1)","family":"zerobench-subquestions-pass-1-zerobench","signalToNoise":3.992,"beta":0.834,"sigma":0.209,"communality":0.751,"difficulty":172.2,"nModels":55,"precision":6.3,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"mathvista-mini","name":"MathVista (mini)","family":"mathvista-mini","signalToNoise":3.985,"beta":1.098,"sigma":0.276,"communality":0.823,"difficulty":116.0,"nModels":115,"precision":13.3,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"winogrande","name":"WinoGrande","family":"winogrande","signalToNoise":3.984,"beta":0.867,"sigma":0.217,"communality":0.742,"difficulty":102.4,"nModels":121,"precision":15.7,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"ale-bench","name":"ALE-Bench","family":"ale-bench","signalToNoise":3.982,"beta":2.319,"sigma":0.582,"communality":0.674,"difficulty":151.0,"nModels":122,"precision":6.3,"precisionMidpoint":6.3,"signalPercentile":32},{"slug":"mathvision-qwen-corrected","name":"MathVision - Qwen corrected annotations","family":"mathvision-qwen-corrected","signalToNoise":3.954,"beta":1.503,"sigma":0.38,"communality":0.173,"difficulty":120.4,"nModels":9,"precision":14.1,"precisionMidpoint":6.4,"signalPercentile":31},{"slug":"countbench","name":"CountBench","family":"countbench","signalToNoise":3.923,"beta":1.11,"sigma":0.283,"communality":0.544,"difficulty":85.8,"nModels":44,"precision":21.6,"precisionMidpoint":6.4,"signalPercentile":31},{"slug":"balrog","name":"BALROG","family":"balrog","signalToNoise":3.899,"beta":1.111,"sigma":0.285,"communality":0.83,"difficulty":154.4,"nModels":37,"precision":6.8,"precisionMidpoint":6.5,"signalPercentile":31},{"slug":"morebench","name":"MoReBench","family":"morebench","signalToNoise":3.894,"beta":0.59,"sigma":0.152,"communality":0.424,"difficulty":99.1,"nModels":14,"precision":9.1,"precisionMidpoint":6.5,"signalPercentile":31},{"slug":"muirbench","name":"MuirBench","family":"muirbench","signalToNoise":3.881,"beta":0.771,"sigma":0.199,"communality":0.759,"difficulty":108.7,"nModels":27,"precision":9.8,"precisionMidpoint":6.5,"signalPercentile":31},{"slug":"theoremqa","name":"TheoremQA","family":"theoremqa","signalToNoise":3.861,"beta":0.84,"sigma":0.218,"communality":0.824,"difficulty":133.2,"nModels":47,"precision":7.7,"precisionMidpoint":6.5,"signalPercentile":31},{"slug":"voxelbench-image-rating","name":"VoxelBench Image Rating","family":"voxelbench-image-rating","signalToNoise":3.86,"beta":2.464,"sigma":0.638,"communality":0.622,"difficulty":156.2,"nModels":10,"precision":6.5,"precisionMidpoint":6.5,"signalPercentile":31},{"slug":"step-game-trueskill-mu","name":"Step Game TrueSkill Mu","family":"step-game-trueskill-mu","signalToNoise":3.83,"beta":2.333,"sigma":0.609,"communality":0.651,"difficulty":137.7,"nModels":57,"precision":6.6,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"mmlongbench-doc","name":"MMLongBench-Doc","family":"mmlongbench-doc","signalToNoise":3.827,"beta":0.896,"sigma":0.234,"communality":0.791,"difficulty":139.8,"nModels":50,"precision":7.6,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"longform-creative-writing","name":"Longform Creative Writing","family":"longform-creative-writing","signalToNoise":3.824,"beta":1.334,"sigma":0.349,"communality":0.77,"difficulty":137.2,"nModels":121,"precision":8.8,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"nimble-web-context","name":"Nimble Web Context","family":"nimble-web-context","signalToNoise":3.821,"beta":0.809,"sigma":0.212,"communality":0.164,"difficulty":128.3,"nModels":11,"precision":8.0,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"eq-bench-v2","name":"EQ-Bench v2","family":"eq-bench-v2","signalToNoise":3.82,"beta":1.249,"sigma":0.327,"communality":0.763,"difficulty":91.0,"nModels":98,"precision":22.6,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"judgemark-v4","name":"Judgemark v4","family":"judgemark-v4","signalToNoise":3.816,"beta":2.248,"sigma":0.589,"communality":0.492,"difficulty":148.3,"nModels":41,"precision":9.1,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"scale-seal-visual-language-understanding","name":"VISTA","family":"scale-seal-visual-language-understanding","signalToNoise":3.811,"beta":0.698,"sigma":0.183,"communality":0.801,"difficulty":150.4,"nModels":52,"precision":6.8,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"seal-0","name":"Seal-0","family":"seal-0","signalToNoise":3.808,"beta":2.546,"sigma":0.669,"communality":0.493,"difficulty":156.2,"nModels":26,"precision":7.8,"precisionMidpoint":6.6,"signalPercentile":30},{"slug":"medqa","name":"MedQA","family":"medqa","signalToNoise":3.782,"beta":1.3,"sigma":0.344,"communality":0.575,"difficulty":111.1,"nModels":58,"precision":17.8,"precisionMidpoint":6.7,"signalPercentile":29},{"slug":"astabench-2026-super-expert","name":"SUPER-Expert (AstaBench 2026)","family":"astabench-2026-super-expert","signalToNoise":3.778,"beta":1.315,"sigma":0.348,"communality":0.691,"difficulty":167.9,"nModels":13,"precision":6.7,"precisionMidpoint":6.7,"signalPercentile":29},{"slug":"tau2-bench-retail","name":"\u03c4\u00b2-bench (Retail)","family":"tau2-bench-retail","signalToNoise":3.767,"beta":1.618,"sigma":0.429,"communality":0.82,"difficulty":130.0,"nModels":71,"precision":12.1,"precisionMidpoint":6.7,"signalPercentile":29},{"slug":"agentvqa","name":"AgentVQA","family":"agentvqa","signalToNoise":3.766,"beta":0.889,"sigma":0.236,"communality":0.53,"difficulty":177.8,"nModels":16,"precision":8.9,"precisionMidpoint":6.7,"signalPercentile":29},{"slug":"c3","name":"C3","family":"c3","signalToNoise":3.742,"beta":0.533,"sigma":0.142,"communality":0.695,"difficulty":57.6,"nModels":11,"precision":11.9,"precisionMidpoint":6.7,"signalPercentile":29},{"slug":"multipl-e-humaneval-bash","name":"MultiPL-E HumanEval (Bash)","family":"multipl-e-humaneval-bash","signalToNoise":3.716,"beta":1.383,"sigma":0.372,"communality":0.661,"difficulty":124.0,"nModels":18,"precision":12.5,"precisionMidpoint":6.8,"signalPercentile":29},{"slug":"hallusionbench","name":"HallusionBench (OpenVLM)","family":"hallusionbench-vlmevalkit","signalToNoise":3.713,"beta":0.49,"sigma":0.132,"communality":0.788,"difficulty":112.7,"nModels":46,"precision":8.0,"precisionMidpoint":6.8,"signalPercentile":29},{"slug":"qodo-code-review-benchmark-125","name":"Qodo AI Code Review Benchmark (125 PRs)","family":"qodo-code-review-benchmark-125","signalToNoise":3.689,"beta":1.105,"sigma":0.299,"communality":0.514,"difficulty":166.3,"nModels":11,"precision":6.8,"precisionMidpoint":6.8,"signalPercentile":28},{"slug":"eval-connections-one-shot-traps","name":"Eval Connections One-Shot with Traps","family":"eval-connections-one-shot-traps","signalToNoise":3.684,"beta":1.875,"sigma":0.509,"communality":0.571,"difficulty":144.2,"nModels":58,"precision":9.8,"precisionMidpoint":6.8,"signalPercentile":28},{"slug":"lvbench","name":"LVBench","family":"lvbench","signalToNoise":3.68,"beta":1.173,"sigma":0.319,"communality":0.718,"difficulty":141.4,"nModels":66,"precision":9.8,"precisionMidpoint":6.8,"signalPercentile":28},{"slug":"cluewsc","name":"CLUEWSC","family":"cluewsc","signalToNoise":3.663,"beta":0.821,"sigma":0.224,"communality":0.609,"difficulty":95.2,"nModels":30,"precision":18.1,"precisionMidpoint":6.9,"signalPercentile":28},{"slug":"mrcr-v2-8-needle-context-arena-128k-auc","name":"MRCR v2 (8-needle, Context Arena) - 128K AUC","family":"mrcr-v2-8-needle-context-arena-128k-auc","signalToNoise":3.662,"beta":2.716,"sigma":0.742,"communality":0.42,"difficulty":151.6,"nModels":68,"precision":9.7,"precisionMidpoint":6.9,"signalPercentile":28},{"slug":"chessbench","name":"ChessBench","family":"chessbench","signalToNoise":3.655,"beta":1.583,"sigma":0.433,"communality":0.605,"difficulty":160.0,"nModels":14,"precision":7.3,"precisionMidpoint":6.9,"signalPercentile":28},{"slug":"medscribe","name":"MedScribe","family":"medscribe","signalToNoise":3.647,"beta":0.996,"sigma":0.273,"communality":0.573,"difficulty":116.2,"nModels":94,"precision":10.7,"precisionMidpoint":6.9,"signalPercentile":28},{"slug":"nova-63","name":"NOVA-63","family":"nova-63","signalToNoise":3.643,"beta":0.345,"sigma":0.095,"communality":0.814,"difficulty":131.8,"nModels":27,"precision":7.1,"precisionMidpoint":6.9,"signalPercentile":28},{"slug":"mm-mt-bench","name":"MM-MT-Bench","family":"mm-mt-bench","signalToNoise":3.64,"beta":1.16,"sigma":0.319,"communality":0.598,"difficulty":108.5,"nModels":21,"precision":14.6,"precisionMidpoint":6.9,"signalPercentile":27},{"slug":"worldvqa-accuracy","name":"WorldVQA Accuracy","family":"worldvqa-accuracy","signalToNoise":3.639,"beta":1.317,"sigma":0.362,"communality":0.526,"difficulty":163.6,"nModels":17,"precision":7.0,"precisionMidpoint":6.9,"signalPercentile":27},{"slug":"nexus","name":"Nexus Function Calling","family":"nexus","signalToNoise":3.603,"beta":1.109,"sigma":0.308,"communality":0.788,"difficulty":124.2,"nModels":13,"precision":10.6,"precisionMidpoint":7.0,"signalPercentile":27},{"slug":"perception-test-mcqa","name":"Perception Test (MCQA)","family":"perception-test-mcqa","signalToNoise":3.6,"beta":0.644,"sigma":0.179,"communality":0.632,"difficulty":122.9,"nModels":12,"precision":10.6,"precisionMidpoint":7.0,"signalPercentile":27},{"slug":"corp_fin_v2","name":"CorpFin v2","family":"corp_fin_v2","signalToNoise":3.576,"beta":0.664,"sigma":0.186,"communality":0.765,"difficulty":130.7,"nModels":121,"precision":7.9,"precisionMidpoint":7.0,"signalPercentile":27},{"slug":"audio-multichallenge-text-output","name":"Audio MultiChallenge (Text Output)","family":"audio-multichallenge-text-output","signalToNoise":3.554,"beta":1.071,"sigma":0.301,"communality":0.831,"difficulty":154.7,"nModels":16,"precision":7.4,"precisionMidpoint":7.1,"signalPercentile":27},{"slug":"advancedif","name":"AdvancedIF","family":"advancedif","signalToNoise":3.546,"beta":1.094,"sigma":0.309,"communality":0.465,"difficulty":132.7,"nModels":40,"precision":9.4,"precisionMidpoint":7.1,"signalPercentile":27},{"slug":"mmmu-pro-vals-4-option","name":"MMMU-Pro (Vals 4-Option)","family":"mmmu-pro-vals-4-option","signalToNoise":3.535,"beta":1.164,"sigma":0.329,"communality":0.683,"difficulty":128.7,"nModels":83,"precision":12.2,"precisionMidpoint":7.1,"signalPercentile":26},{"slug":"alignbench-v1-1","name":"AlignBench v1.1","family":"alignbench-v1-1","signalToNoise":3.529,"beta":0.92,"sigma":0.261,"communality":0.753,"difficulty":91.9,"nModels":40,"precision":15.0,"precisionMidpoint":7.1,"signalPercentile":26},{"slug":"omni-math-helm-capabilities","name":"Omni-MATH (HELM Capabilities)","family":"omni-math-helm-capabilities","signalToNoise":3.511,"beta":1.27,"sigma":0.362,"communality":0.742,"difficulty":138.8,"nModels":56,"precision":9.2,"precisionMidpoint":7.2,"signalPercentile":26},{"slug":"hle-verified-1811","name":"Humanity's Last Exam - full verified 1,811-item set","family":"hle-verified-1811","signalToNoise":3.506,"beta":0.936,"sigma":0.267,"communality":0.417,"difficulty":161.2,"nModels":12,"precision":7.2,"precisionMidpoint":7.2,"signalPercentile":26},{"slug":"llm-chess-elo","name":"LLM Chess Elo","family":"llm-chess-elo","signalToNoise":3.486,"beta":1.668,"sigma":0.479,"communality":0.784,"difficulty":145.9,"nModels":94,"precision":7.2,"precisionMidpoint":7.2,"signalPercentile":26},{"slug":"multipl-e-humaneval-php","name":"MultiPL-E HumanEval (PHP)","family":"multipl-e-humaneval-php","signalToNoise":3.478,"beta":1.258,"sigma":0.362,"communality":0.749,"difficulty":105.6,"nModels":18,"precision":18.1,"precisionMidpoint":7.2,"signalPercentile":26},{"slug":"astabench-2026-e2e-hard","name":"E2E-Bench-Hard (AstaBench 2026)","family":"astabench-2026-e2e-hard","signalToNoise":3.455,"beta":1.969,"sigma":0.57,"communality":0.687,"difficulty":161.2,"nModels":13,"precision":7.5,"precisionMidpoint":7.3,"signalPercentile":26},{"slug":"perceptionbench","name":"PerceptionBench","family":"perceptionbench","signalToNoise":3.431,"beta":0.974,"sigma":0.284,"communality":0.524,"difficulty":156.4,"nModels":17,"precision":7.5,"precisionMidpoint":7.3,"signalPercentile":26},{"slug":"buseybench-svg","name":"BuseyBench SVG","family":"buseybench-svg","signalToNoise":3.424,"beta":1.261,"sigma":0.368,"communality":0.73,"difficulty":157.3,"nModels":249,"precision":7.6,"precisionMidpoint":7.4,"signalPercentile":25},{"slug":"eq-bench-3","name":"EQ-Bench 3","family":"eq-bench-3","signalToNoise":3.411,"beta":1.646,"sigma":0.482,"communality":0.782,"difficulty":138.9,"nModels":76,"precision":7.4,"precisionMidpoint":7.4,"signalPercentile":25},{"slug":"creative-writing-v3-elo","name":"Creative Writing v3 Elo","family":"creative-writing-v3-elo","signalToNoise":3.38,"beta":1.409,"sigma":0.417,"communality":0.835,"difficulty":142.8,"nModels":121,"precision":7.4,"precisionMidpoint":7.4,"signalPercentile":25},{"slug":"textvqa-val","name":"TextVQA (validation)","family":"textvqa-val","signalToNoise":3.366,"beta":0.562,"sigma":0.167,"communality":0.716,"difficulty":51.7,"nModels":21,"precision":14.7,"precisionMidpoint":7.5,"signalPercentile":25},{"slug":"officeqa-pro-card","name":"OfficeQA Pro","family":"officeqa-pro-card","signalToNoise":3.361,"beta":1.717,"sigma":0.511,"communality":0.227,"difficulty":154.9,"nModels":12,"precision":8.2,"precisionMidpoint":7.5,"signalPercentile":25},{"slug":"benchcad-reported-iou","name":"BenchCAD - geometric overlap","family":"benchcad-reported-iou","signalToNoise":3.355,"beta":2.233,"sigma":0.666,"communality":0.342,"difficulty":157.8,"nModels":12,"precision":8.6,"precisionMidpoint":7.5,"signalPercentile":25},{"slug":"scipredict-scale-ai-2","name":"SciPredict","family":"scipredict-scale-ai-2","signalToNoise":3.342,"beta":0.476,"sigma":0.143,"communality":0.241,"difficulty":218.2,"nModels":15,"precision":8.4,"precisionMidpoint":7.5,"signalPercentile":25},{"slug":"hlab-v1-1","name":"HLAB-Vals-1.1","family":"hlab-v1-1","signalToNoise":3.332,"beta":1.353,"sigma":0.406,"communality":0.103,"difficulty":201.4,"nModels":32,"precision":11.0,"precisionMidpoint":7.6,"signalPercentile":24},{"slug":"culturalriddles","name":"CulturalRiddles","family":"culturalriddles","signalToNoise":3.318,"beta":1.593,"sigma":0.48,"communality":0.781,"difficulty":146.1,"nModels":14,"precision":9.4,"precisionMidpoint":7.6,"signalPercentile":24},{"slug":"eval-connections","name":"Eval Connections","family":"eval-connections","signalToNoise":3.317,"beta":3.122,"sigma":0.941,"communality":0.305,"difficulty":138.2,"nModels":80,"precision":23.6,"precisionMidpoint":7.6,"signalPercentile":24},{"slug":"simplevqa","name":"SimpleVQA","family":"simplevqa","signalToNoise":3.289,"beta":0.759,"sigma":0.231,"communality":0.771,"difficulty":133.6,"nModels":32,"precision":8.7,"precisionMidpoint":7.7,"signalPercentile":24},{"slug":"countbenchqa","name":"CountBenchQA","family":"countbenchqa","signalToNoise":3.27,"beta":0.956,"sigma":0.292,"communality":0.785,"difficulty":75.1,"nModels":10,"precision":23.5,"precisionMidpoint":7.7,"signalPercentile":24},{"slug":"roboflow-vision-evals","name":"Roboflow Vision Evals","family":"roboflow-vision-evals","signalToNoise":3.267,"beta":0.971,"sigma":0.297,"communality":0.352,"difficulty":120.5,"nModels":20,"precision":11.3,"precisionMidpoint":7.7,"signalPercentile":24},{"slug":"bridge-medical-zero-shot","name":"BRIDGE Medical (zero-shot)","family":"bridge-medical-zero-shot","signalToNoise":3.253,"beta":0.546,"sigma":0.168,"communality":0.828,"difficulty":152.1,"nModels":62,"precision":7.9,"precisionMidpoint":7.7,"signalPercentile":24},{"slug":"tutorbench-scale-ai-2","name":"TutorBench","family":"tutorbench-scale-ai-2","signalToNoise":3.223,"beta":0.538,"sigma":0.167,"communality":0.434,"difficulty":144.3,"nModels":20,"precision":8.1,"precisionMidpoint":7.8,"signalPercentile":23},{"slug":"spanish-scale-ai-2","name":"Spanish","family":"spanish-scale-ai-2","signalToNoise":3.22,"beta":2.177,"sigma":0.676,"communality":0.561,"difficulty":127.9,"nModels":22,"precision":7.8,"precisionMidpoint":7.8,"signalPercentile":23},{"slug":"taxbench-total-pass1","name":"TaxBench Total pass@1","family":"taxbench-total-pass1","signalToNoise":3.22,"beta":0.816,"sigma":0.253,"communality":0.436,"difficulty":130.4,"nModels":14,"precision":9.3,"precisionMidpoint":7.8,"signalPercentile":23},{"slug":"financeqa","name":"FinanceQA","family":"financeqa","signalToNoise":3.214,"beta":0.817,"sigma":0.254,"communality":0.502,"difficulty":149.8,"nModels":19,"precision":8.2,"precisionMidpoint":7.8,"signalPercentile":23},{"slug":"sweeper-bench","name":"SWEeper-Bench","family":"sweeper-bench","signalToNoise":3.209,"beta":0.944,"sigma":0.294,"communality":0.164,"difficulty":162.0,"nModels":12,"precision":7.9,"precisionMidpoint":7.8,"signalPercentile":23},{"slug":"aime-2024","name":"AIME 2024","family":"aime-2024","signalToNoise":3.197,"beta":2.319,"sigma":0.725,"communality":0.695,"difficulty":131.7,"nModels":106,"precision":29.8,"precisionMidpoint":7.9,"signalPercentile":23},{"slug":"bool-q","name":"BoolQ","family":"bool-q","signalToNoise":3.185,"beta":0.856,"sigma":0.269,"communality":0.619,"difficulty":86.0,"nModels":55,"precision":24.4,"precisionMidpoint":7.9,"signalPercentile":23},{"slug":"chinese-scale-ai-2","name":"Chinese","family":"chinese-scale-ai-2","signalToNoise":3.182,"beta":2.045,"sigma":0.643,"communality":0.614,"difficulty":128.2,"nModels":21,"precision":7.9,"precisionMidpoint":7.9,"signalPercentile":23},{"slug":"shroombench","name":"Shroombench (mushroom identification, top-1)","family":"shroombench","signalToNoise":3.181,"beta":1.5,"sigma":0.472,"communality":0.259,"difficulty":165.9,"nModels":21,"precision":7.9,"precisionMidpoint":7.9,"signalPercentile":22},{"slug":"unitybench-mini3d","name":"UnityBench-Mini3D","family":"unitybench-mini3d","signalToNoise":3.181,"beta":1.562,"sigma":0.491,"communality":0.225,"difficulty":153.0,"nModels":13,"precision":8.8,"precisionMidpoint":7.9,"signalPercentile":22},{"slug":"labbench2","name":"LABBench2","family":"labbench2","signalToNoise":3.179,"beta":0.973,"sigma":0.306,"communality":0.0,"difficulty":120.8,"nModels":11,"precision":11.5,"precisionMidpoint":7.9,"signalPercentile":22},{"slug":"policybench-us-exact-match","name":"PolicyBench US Exact Match","family":"policybench-us-exact-match","signalToNoise":3.163,"beta":0.797,"sigma":0.252,"communality":0.294,"difficulty":114.6,"nModels":23,"precision":10.9,"precisionMidpoint":8.0,"signalPercentile":22},{"slug":"astabench-2026-discoverybench","name":"DiscoveryBench (AstaBench 2026)","family":"astabench-2026-discoverybench","signalToNoise":3.16,"beta":0.678,"sigma":0.215,"communality":0.603,"difficulty":181.4,"nModels":13,"precision":8.1,"precisionMidpoint":8.0,"signalPercentile":22},{"slug":"creative-writing-v2","name":"Creative Writing v2","family":"creative-writing-v2","signalToNoise":3.147,"beta":0.876,"sigma":0.278,"communality":0.761,"difficulty":91.0,"nModels":53,"precision":16.2,"precisionMidpoint":8.0,"signalPercentile":22},{"slug":"professional-reasoning-benchmark-finance-scale-ai-2","name":"Professional Reasoning Benchmark - Finance","family":"professional-reasoning-benchmark-finance-scale-ai-2","signalToNoise":3.145,"beta":0.652,"sigma":0.207,"communality":0.625,"difficulty":159.4,"nModels":38,"precision":8.0,"precisionMidpoint":8.0,"signalPercentile":22},{"slug":"race-middle","name":"RACE-Middle","family":"race-middle","signalToNoise":3.136,"beta":0.426,"sigma":0.136,"communality":0.863,"difficulty":89.2,"nModels":27,"precision":11.8,"precisionMidpoint":8.0,"signalPercentile":21},{"slug":"mmvet-vlmevalkit","name":"MM-Vet (OpenVLM)","family":"mmvet-vlmevalkit","signalToNoise":3.129,"beta":0.788,"sigma":0.252,"communality":0.772,"difficulty":100.7,"nModels":57,"precision":13.1,"precisionMidpoint":8.0,"signalPercentile":21},{"slug":"live-bench","name":"LiveBench","family":"live-bench","signalToNoise":3.12,"beta":0.932,"sigma":0.299,"communality":0.788,"difficulty":120.7,"nModels":86,"precision":11.6,"precisionMidpoint":8.1,"signalPercentile":21},{"slug":"quant-arena","name":"Quant-Arena (iterative total score)","family":"quant-arena","signalToNoise":3.118,"beta":0.62,"sigma":0.199,"communality":0.468,"difficulty":177.8,"nModels":9,"precision":8.1,"precisionMidpoint":8.1,"signalPercentile":21},{"slug":"multipl-e-humaneval-java","name":"MultiPL-E HumanEval (Java)","family":"multipl-e-humaneval-java","signalToNoise":3.114,"beta":1.21,"sigma":0.388,"communality":0.713,"difficulty":102.9,"nModels":18,"precision":20.4,"precisionMidpoint":8.1,"signalPercentile":21},{"slug":"warc-bench-test-192","name":"WARC-Bench (192-task test split)","family":"warc-bench-test-192","signalToNoise":3.079,"beta":1.673,"sigma":0.544,"communality":0.378,"difficulty":143.2,"nModels":8,"precision":11.6,"precisionMidpoint":8.2,"signalPercentile":21},{"slug":"math","name":"MATH","family":"math","signalToNoise":3.07,"beta":1.676,"sigma":0.546,"communality":0.756,"difficulty":111.6,"nModels":201,"precision":27.7,"precisionMidpoint":8.2,"signalPercentile":21},{"slug":"trivia-qa","name":"TriviaQA","family":"trivia-qa","signalToNoise":3.069,"beta":0.902,"sigma":0.294,"communality":0.822,"difficulty":76.8,"nModels":58,"precision":22.0,"precisionMidpoint":8.2,"signalPercentile":21},{"slug":"simpleqa-verified-epoch-anti-abstention","name":"SimpleQA Verified (Epoch anti-abstention)","family":"simpleqa-verified-epoch-anti-abstention","signalToNoise":3.064,"beta":1.421,"sigma":0.464,"communality":0.629,"difficulty":161.0,"nModels":72,"precision":8.6,"precisionMidpoint":8.2,"signalPercentile":20},{"slug":"longvideobench-reported","name":"LongVideoBench - release-reported accuracy","family":"longvideobench-reported","signalToNoise":3.039,"beta":0.653,"sigma":0.215,"communality":0.81,"difficulty":110.3,"nModels":11,"precision":10.7,"precisionMidpoint":8.3,"signalPercentile":20},{"slug":"frames","name":"FRAMES","family":"frames","signalToNoise":3.027,"beta":0.917,"sigma":0.303,"communality":0.364,"difficulty":99.7,"nModels":17,"precision":15.9,"precisionMidpoint":8.3,"signalPercentile":20},{"slug":"japanese-scale-ai-2","name":"Japanese","family":"japanese-scale-ai-2","signalToNoise":3.022,"beta":1.85,"sigma":0.612,"communality":0.664,"difficulty":127.5,"nModels":17,"precision":8.3,"precisionMidpoint":8.3,"signalPercentile":20},{"slug":"facts-benchmark-suite","name":"FACTS Benchmark Suite","family":"facts-benchmark-suite","signalToNoise":3.014,"beta":1.305,"sigma":0.433,"communality":0.528,"difficulty":148.9,"nModels":28,"precision":9.4,"precisionMidpoint":8.4,"signalPercentile":20},{"slug":"astabench-2026-arxivdigestables","name":"ArxivDIGESTables-Clean (AstaBench 2026)","family":"astabench-2026-arxivdigestables","signalToNoise":3.002,"beta":0.578,"sigma":0.193,"communality":0.568,"difficulty":188.6,"nModels":13,"precision":8.6,"precisionMidpoint":8.4,"signalPercentile":20},{"slug":"studentbench-lesson-planning","name":"ATLAS-Education (StudentBench) Lesson Planning","family":"studentbench-lesson-planning","signalToNoise":2.988,"beta":2.168,"sigma":0.726,"communality":0.441,"difficulty":155.4,"nModels":12,"precision":8.4,"precisionMidpoint":8.4,"signalPercentile":20},{"slug":"hellaswag-openllm-v1-harness","name":"HellaSwag (Open LLM Leaderboard v1)","family":"hellaswag-openllm-v1-harness","signalToNoise":2.972,"beta":0.907,"sigma":0.305,"communality":0.729,"difficulty":70.4,"nModels":118,"precision":29.5,"precisionMidpoint":8.5,"signalPercentile":19},{"slug":"mathvista-vlmevalkit","name":"MathVista (OpenVLM)","family":"mathvista-vlmevalkit","signalToNoise":2.957,"beta":0.785,"sigma":0.266,"communality":0.767,"difficulty":104.2,"nModels":58,"precision":13.2,"precisionMidpoint":8.5,"signalPercentile":19},{"slug":"obviousbench","name":"ObviousBench","family":"obviousbench","signalToNoise":2.952,"beta":1.781,"sigma":0.603,"communality":0.417,"difficulty":118.5,"nModels":212,"precision":28.2,"precisionMidpoint":8.5,"signalPercentile":19},{"slug":"tax_eval_v2","name":"TaxEval v2","family":"tax_eval_v2","signalToNoise":2.936,"beta":0.526,"sigma":0.179,"communality":0.675,"difficulty":102.6,"nModels":126,"precision":10.6,"precisionMidpoint":8.6,"signalPercentile":19},{"slug":"simpleqa","name":"SimpleQA","family":"simpleqa","signalToNoise":2.927,"beta":1.456,"sigma":0.497,"communality":0.605,"difficulty":153.0,"nModels":69,"precision":9.5,"precisionMidpoint":8.6,"signalPercentile":19},{"slug":"concept-synth-abd-v1-2","name":"Concept Synth ABD v1.2","family":"concept-synth-abd-v1-2","signalToNoise":2.917,"beta":1.539,"sigma":0.528,"communality":0.053,"difficulty":110.3,"nModels":9,"precision":26.1,"precisionMidpoint":8.6,"signalPercentile":19},{"slug":"ma-bench","name":"MA-Bench","family":"ma-bench","signalToNoise":2.909,"beta":0.64,"sigma":0.22,"communality":0.395,"difficulty":158.3,"nModels":17,"precision":8.7,"precisionMidpoint":8.7,"signalPercentile":19},{"slug":"multipl-e-humaneval-cpp","name":"MultiPL-E HumanEval (C++)","family":"multipl-e-humaneval-cpp","signalToNoise":2.9,"beta":1.25,"sigma":0.431,"communality":0.663,"difficulty":104.0,"nModels":18,"precision":22.4,"precisionMidpoint":8.7,"signalPercentile":19},{"slug":"egoschema","name":"EgoSchema","family":"egoschema","signalToNoise":2.893,"beta":0.898,"sigma":0.31,"communality":0.439,"difficulty":113.3,"nModels":18,"precision":15.0,"precisionMidpoint":8.7,"signalPercentile":18},{"slug":"pubmedqa","name":"PubMedQA","family":"pubmedqa","signalToNoise":2.857,"beta":0.434,"sigma":0.152,"communality":0.557,"difficulty":100.2,"nModels":9,"precision":13.4,"precisionMidpoint":8.8,"signalPercentile":18},{"slug":"drop","name":"DROP","family":"drop","signalToNoise":2.856,"beta":0.837,"sigma":0.293,"communality":0.781,"difficulty":82.7,"nModels":64,"precision":19.3,"precisionMidpoint":8.8,"signalPercentile":18},{"slug":"vending-bench-days-until-sales-stop-percentage","name":"Vending-Bench","family":"vending-bench","signalToNoise":2.845,"beta":2.611,"sigma":0.918,"communality":0.282,"difficulty":128.8,"nModels":21,"precision":34.9,"precisionMidpoint":8.8,"signalPercentile":18},{"slug":"babyvision","name":"BabyVision","family":"babyvision","signalToNoise":2.823,"beta":1.652,"sigma":0.585,"communality":0.42,"difficulty":160.6,"nModels":44,"precision":9.2,"precisionMidpoint":8.9,"signalPercentile":18},{"slug":"blink","name":"BLINK","family":"blink","signalToNoise":2.819,"beta":0.542,"sigma":0.192,"communality":0.769,"difficulty":107.2,"nModels":43,"precision":11.1,"precisionMidpoint":8.9,"signalPercentile":18},{"slug":"toolsandbox","name":"ToolSandbox","family":"toolsandbox","signalToNoise":2.819,"beta":0.704,"sigma":0.25,"communality":0.312,"difficulty":95.5,"nModels":8,"precision":14.3,"precisionMidpoint":8.9,"signalPercentile":18},{"slug":"medcode","name":"MedCode","family":"medcode","signalToNoise":2.818,"beta":0.713,"sigma":0.253,"communality":0.539,"difficulty":163.6,"nModels":94,"precision":9.0,"precisionMidpoint":8.9,"signalPercentile":17},{"slug":"veracode-genai-code-security","name":"Veracode GenAI Code Security","family":"veracode-genai-code-security","signalToNoise":2.801,"beta":0.928,"sigma":0.331,"communality":0.194,"difficulty":135.7,"nModels":9,"precision":10.7,"precisionMidpoint":9.0,"signalPercentile":17},{"slug":"hellaswag-unspecified-scoring-protocol","name":"HellaSwag (Unspecified Scoring Protocol)","family":"hellaswag-unspecified-scoring-protocol","signalToNoise":2.794,"beta":0.878,"sigma":0.314,"communality":0.697,"difficulty":74.6,"nModels":118,"precision":27.8,"precisionMidpoint":9.0,"signalPercentile":17},{"slug":"seed-bench-image","name":"SEED-Bench (image)","family":"seed-bench-image","signalToNoise":2.773,"beta":0.341,"sigma":0.123,"communality":0.643,"difficulty":65.6,"nModels":15,"precision":13.6,"precisionMidpoint":9.1,"signalPercentile":17},{"slug":"llm-divergent-thinking-score","name":"LLM Divergent Thinking Score","family":"llm-divergent-thinking-score","signalToNoise":2.772,"beta":0.325,"sigma":0.117,"communality":0.245,"difficulty":152.2,"nModels":17,"precision":9.1,"precisionMidpoint":9.1,"signalPercentile":17},{"slug":"cmmlu","name":"CMMLU","family":"cmmlu","signalToNoise":2.771,"beta":1.367,"sigma":0.493,"communality":0.653,"difficulty":99.4,"nModels":55,"precision":35.0,"precisionMidpoint":9.1,"signalPercentile":17},{"slug":"snorkel-legal","name":"SnorkelLegal","family":"snorkel-legal","signalToNoise":2.768,"beta":0.955,"sigma":0.345,"communality":0.0,"difficulty":185.4,"nModels":13,"precision":9.6,"precisionMidpoint":9.1,"signalPercentile":17},{"slug":"bfcl-v3-overall-accuracy","name":"BFCL v3 Overall Accuracy","family":"bfcl-v3-overall-accuracy","signalToNoise":2.767,"beta":0.742,"sigma":0.268,"communality":0.743,"difficulty":113.2,"nModels":83,"precision":12.2,"precisionMidpoint":9.1,"signalPercentile":17},{"slug":"radle-2","name":"Radiology's Last Exam 2.0 (RadLE-C)","family":"radle-2","signalToNoise":2.761,"beta":0.736,"sigma":0.267,"communality":0.591,"difficulty":187.7,"nModels":9,"precision":9.5,"precisionMidpoint":9.1,"signalPercentile":16},{"slug":"writingbench","name":"WritingBench","family":"writingbench","signalToNoise":2.761,"beta":0.934,"sigma":0.338,"communality":0.673,"difficulty":104.4,"nModels":64,"precision":16.3,"precisionMidpoint":9.1,"signalPercentile":16},{"slug":"slake","name":"SLAKE","family":"slake","signalToNoise":2.755,"beta":0.662,"sigma":0.24,"communality":0.623,"difficulty":97.1,"nModels":23,"precision":13.9,"precisionMidpoint":9.1,"signalPercentile":16},{"slug":"lambada","name":"LAMBADA","family":"lambada","signalToNoise":2.74,"beta":0.473,"sigma":0.173,"communality":0.853,"difficulty":32.7,"nModels":34,"precision":17.7,"precisionMidpoint":9.2,"signalPercentile":16},{"slug":"txbench-pp-v2","name":"TxBench Preclinical Pharmacology V1 Clean Reruns","family":"txbench-pp-v2","signalToNoise":2.73,"beta":1.065,"sigma":0.39,"communality":0.137,"difficulty":159.5,"nModels":24,"precision":9.4,"precisionMidpoint":9.2,"signalPercentile":16},{"slug":"roboflow-vision-evals-2026-09","name":"Roboflow Vision Evals (September 2026 protocol)","family":"roboflow-vision-evals-2026-09","signalToNoise":2.717,"beta":0.881,"sigma":0.324,"communality":0.259,"difficulty":125.7,"nModels":41,"precision":12.0,"precisionMidpoint":9.3,"signalPercentile":16},{"slug":"redlinebench","name":"RedlineBench","family":"redlinebench","signalToNoise":2.716,"beta":0.537,"sigma":0.198,"communality":0.066,"difficulty":170.2,"nModels":26,"precision":9.3,"precisionMidpoint":9.3,"signalPercentile":16},{"slug":"ifeval","name":"IFEval","family":"ifeval","signalToNoise":2.692,"beta":1.02,"sigma":0.379,"communality":0.61,"difficulty":90.3,"nModels":145,"precision":23.2,"precisionMidpoint":9.3,"signalPercentile":15},{"slug":"professional-reasoning-benchmark-legal-scale-ai-2","name":"Professional Reasoning Benchmark - Legal","family":"professional-reasoning-benchmark-legal-scale-ai-2","signalToNoise":2.685,"beta":0.58,"sigma":0.216,"communality":0.549,"difficulty":161.1,"nModels":43,"precision":9.4,"precisionMidpoint":9.4,"signalPercentile":15},{"slug":"browsecomp-long-context-128k","name":"BrowseComp Long Context - 128K","family":"browsecomp-long-context-128k","signalToNoise":2.676,"beta":0.759,"sigma":0.284,"communality":0.0,"difficulty":75.2,"nModels":10,"precision":20.2,"precisionMidpoint":9.4,"signalPercentile":15},{"slug":"legacy-bench-fireworks","name":"Legacy-Bench (Fireworks, avg@3)","family":"legacy-bench-fireworks","signalToNoise":2.658,"beta":0.579,"sigma":0.218,"communality":0.0,"difficulty":152.7,"nModels":12,"precision":9.6,"precisionMidpoint":9.5,"signalPercentile":15},{"slug":"snorkel-underwrite","name":"SnorkelUnderwrite","family":"snorkel-underwrite","signalToNoise":2.647,"beta":1.435,"sigma":0.542,"communality":0.642,"difficulty":130.6,"nModels":32,"precision":15.5,"precisionMidpoint":9.5,"signalPercentile":15},{"slug":"llm-creative-story-writing","name":"LLM Creative Story Writing","family":"llm-creative-story-writing","signalToNoise":2.64,"beta":1.978,"sigma":0.749,"communality":0.435,"difficulty":156.0,"nModels":50,"precision":9.5,"precisionMidpoint":9.5,"signalPercentile":15},{"slug":"mm-ifeval","name":"MM-IFEval","family":"mm-ifeval","signalToNoise":2.629,"beta":0.812,"sigma":0.309,"communality":0.821,"difficulty":107.7,"nModels":14,"precision":14.6,"precisionMidpoint":9.6,"signalPercentile":15},{"slug":"activitynet-qa","name":"ActivityNet-QA","family":"activitynet-qa","signalToNoise":2.598,"beta":0.556,"sigma":0.214,"communality":0.53,"difficulty":113.7,"nModels":17,"precision":11.5,"precisionMidpoint":9.7,"signalPercentile":15},{"slug":"chinese-simpleqa","name":"Chinese SimpleQA","family":"chinese-simpleqa","signalToNoise":2.598,"beta":0.858,"sigma":0.33,"communality":0.703,"difficulty":117.3,"nModels":22,"precision":13.5,"precisionMidpoint":9.7,"signalPercentile":14},{"slug":"ai2d-vlmevalkit","name":"AI2D (OpenVLM)","family":"ai2d-vlmevalkit","signalToNoise":2.597,"beta":0.684,"sigma":0.263,"communality":0.553,"difficulty":80.9,"nModels":58,"precision":20.3,"precisionMidpoint":9.7,"signalPercentile":14},{"slug":"multi-if","name":"Multi-IF","family":"multi-if","signalToNoise":2.574,"beta":0.971,"sigma":0.377,"communality":0.728,"difficulty":112.0,"nModels":78,"precision":15.9,"precisionMidpoint":9.8,"signalPercentile":14},{"slug":"mme","name":"MME","family":"mme","signalToNoise":2.539,"beta":0.859,"sigma":0.338,"communality":0.468,"difficulty":101.4,"nModels":22,"precision":22.4,"precisionMidpoint":9.9,"signalPercentile":14},{"slug":"kernelbench-hard-normalized-h100","name":"KernelBench Hard Normalized (H100)","family":"kernelbench-hard-normalized-h100","signalToNoise":2.53,"beta":2.13,"sigma":0.842,"communality":0.053,"difficulty":151.7,"nModels":13,"precision":12.5,"precisionMidpoint":9.9,"signalPercentile":14},{"slug":"healthbench-card-hard","name":"HealthBench Hard","family":"healthbench-card-hard","signalToNoise":2.526,"beta":1.226,"sigma":0.485,"communality":0.379,"difficulty":167.7,"nModels":17,"precision":10.0,"precisionMidpoint":10.0,"signalPercentile":14},{"slug":"bfcl-v4-overall-accuracy","name":"BFCL v4 Overall Accuracy","family":"bfcl-v4-overall-accuracy","signalToNoise":2.519,"beta":1.012,"sigma":0.402,"communality":0.758,"difficulty":136.0,"nModels":85,"precision":12.1,"precisionMidpoint":10.0,"signalPercentile":14},{"slug":"aikido-known-cve-detection","name":"Aikido Known-CVE Detection (Pass@3 Union)","family":"aikido-known-cve-detection","signalToNoise":2.507,"beta":1.217,"sigma":0.485,"communality":0.291,"difficulty":135.5,"nModels":13,"precision":13.5,"precisionMidpoint":10.0,"signalPercentile":13},{"slug":"mbpp","name":"MBPP","family":"mbpp","signalToNoise":2.506,"beta":1.154,"sigma":0.461,"communality":0.569,"difficulty":95.6,"nModels":177,"precision":28.0,"precisionMidpoint":10.0,"signalPercentile":13},{"slug":"motionbench-dev","name":"MotionBench (dev)","family":"motionbench-dev","signalToNoise":2.496,"beta":0.615,"sigma":0.246,"communality":0.495,"difficulty":146.1,"nModels":8,"precision":12.9,"precisionMidpoint":10.1,"signalPercentile":13},{"slug":"matplotgym","name":"matplotgym","family":"matplotgym","signalToNoise":2.479,"beta":0.637,"sigma":0.257,"communality":0.475,"difficulty":152.5,"nModels":9,"precision":10.4,"precisionMidpoint":10.2,"signalPercentile":13},{"slug":"multipl-e-humaneval-typescript","name":"MultiPL-E HumanEval (TypeScript)","family":"multipl-e-humaneval-typescript","signalToNoise":2.479,"beta":1.146,"sigma":0.462,"communality":0.61,"difficulty":109.0,"nModels":18,"precision":20.9,"precisionMidpoint":10.2,"signalPercentile":13},{"slug":"odysseys2","name":"Odysseys2","family":"odysseys2","signalToNoise":2.446,"beta":1.863,"sigma":0.762,"communality":0.224,"difficulty":162.1,"nModels":14,"precision":10.5,"precisionMidpoint":10.3,"signalPercentile":13},{"slug":"squad2","name":"SQuAD 2.0","family":"squad2","signalToNoise":2.446,"beta":0.874,"sigma":0.357,"communality":0.801,"difficulty":66.4,"nModels":27,"precision":31.1,"precisionMidpoint":10.3,"signalPercentile":13},{"slug":"txbench-ab-v1-100","name":"TxBench Antibody Discovery V1 (100 evaluations)","family":"txbench-ab-v1-100","signalToNoise":2.426,"beta":1.252,"sigma":0.516,"communality":0.094,"difficulty":172.4,"nModels":24,"precision":10.4,"precisionMidpoint":10.4,"signalPercentile":13},{"slug":"chessbench-ai-elo","name":"ChessBench Elo","family":"chessbench-ai-elo","signalToNoise":2.401,"beta":1.542,"sigma":0.642,"communality":0.601,"difficulty":150.0,"nModels":65,"precision":10.5,"precisionMidpoint":10.5,"signalPercentile":12},{"slug":"pmc-vqa","name":"PMC-VQA","family":"pmc-vqa","signalToNoise":2.393,"beta":0.606,"sigma":0.253,"communality":0.492,"difficulty":157.6,"nModels":23,"precision":13.4,"precisionMidpoint":10.5,"signalPercentile":12},{"slug":"omnidocbench-v1-5","name":"OmniDocBench v1.5","family":"omnidocbench-v1-5","signalToNoise":2.379,"beta":0.664,"sigma":0.279,"communality":0.482,"difficulty":79.3,"nModels":38,"precision":18.7,"precisionMidpoint":10.6,"signalPercentile":12},{"slug":"cmath","name":"CMATH","family":"cmath","signalToNoise":2.374,"beta":1.022,"sigma":0.43,"communality":0.588,"difficulty":80.1,"nModels":29,"precision":32.7,"precisionMidpoint":10.6,"signalPercentile":12},{"slug":"superglue","name":"SuperGLUE","family":"superglue","signalToNoise":2.368,"beta":0.249,"sigma":0.105,"communality":0.931,"difficulty":64.5,"nModels":8,"precision":12.1,"precisionMidpoint":10.6,"signalPercentile":12},{"slug":"roboflow-vision-evals-2026-10","name":"Roboflow Vision Evals (October 2026, five tasks)","family":"roboflow-vision-evals-2026-10","signalToNoise":2.34,"beta":0.896,"sigma":0.383,"communality":0.246,"difficulty":137.5,"nModels":41,"precision":12.4,"precisionMidpoint":10.8,"signalPercentile":12},{"slug":"mlvu-m-avg","name":"MLVU (multiple-choice M-Avg)","family":"mlvu-m-avg","signalToNoise":2.333,"beta":0.749,"sigma":0.321,"communality":0.487,"difficulty":97.7,"nModels":58,"precision":17.7,"precisionMidpoint":10.8,"signalPercentile":12},{"slug":"natural-questions","name":"Natural Questions (open-domain, closed-book)","family":"natural-questions","signalToNoise":2.331,"beta":0.681,"sigma":0.292,"communality":0.737,"difficulty":132.3,"nModels":57,"precision":12.1,"precisionMidpoint":10.8,"signalPercentile":11},{"slug":"concept-synth-induction-v1-ci","name":"Concept Synth INDUCTION v1","family":"concept-synth-induction-v1","signalToNoise":2.329,"beta":1.292,"sigma":0.555,"communality":0.178,"difficulty":136.8,"nModels":12,"precision":14.6,"precisionMidpoint":10.8,"signalPercentile":11},{"slug":"realworldqa-reported","name":"RealWorldQA - release-reported visual perception","family":"realworldqa-reported","signalToNoise":2.316,"beta":0.54,"sigma":0.233,"communality":0.692,"difficulty":85.1,"nModels":91,"precision":15.7,"precisionMidpoint":10.9,"signalPercentile":11},{"slug":"c-eval","name":"C-Eval","family":"c-eval","signalToNoise":2.311,"beta":1.048,"sigma":0.453,"communality":0.71,"difficulty":103.8,"nModels":149,"precision":25.6,"precisionMidpoint":10.9,"signalPercentile":11},{"slug":"vlmsareblind","name":"VLMs Are Blind","family":"vlmsareblind","signalToNoise":2.309,"beta":1.155,"sigma":0.5,"communality":0.491,"difficulty":100.8,"nModels":17,"precision":26.9,"precisionMidpoint":10.9,"signalPercentile":11},{"slug":"deep-research-bench","name":"Deep Research Bench","family":"deep-research-bench","signalToNoise":2.292,"beta":0.442,"sigma":0.193,"communality":0.213,"difficulty":157.0,"nModels":22,"precision":11.0,"precisionMidpoint":11.0,"signalPercentile":11},{"slug":"wordlebench","name":"WordleBench","family":"wordlebench","signalToNoise":2.291,"beta":1.908,"sigma":0.833,"communality":0.323,"difficulty":138.1,"nModels":92,"precision":20.9,"precisionMidpoint":11.0,"signalPercentile":11},{"slug":"ifbench-artificial-analysis","name":"IFBench (Artificial Analysis avg@5)","family":"ifbench-artificial-analysis","signalToNoise":2.285,"beta":0.963,"sigma":0.421,"communality":0.642,"difficulty":137.3,"nModels":324,"precision":13.3,"precisionMidpoint":11.0,"signalPercentile":11},{"slug":"humaneval","name":"HumanEval","family":"humaneval","signalToNoise":2.283,"beta":1.38,"sigma":0.604,"communality":0.599,"difficulty":99.7,"nModels":196,"precision":37.0,"precisionMidpoint":11.0,"signalPercentile":10},{"slug":"ifeval-helm-capabilities","name":"IFEval (HELM Capabilities)","family":"ifeval-helm-capabilities","signalToNoise":2.262,"beta":0.629,"sigma":0.278,"communality":0.504,"difficulty":65.6,"nModels":64,"precision":21.5,"precisionMidpoint":11.1,"signalPercentile":10},{"slug":"screenspot-v2-desktop","name":"ScreenSpot-v2","family":"screenspot-v2","signalToNoise":2.222,"beta":1.174,"sigma":0.528,"communality":0.275,"difficulty":92.1,"nModels":8,"precision":34.3,"precisionMidpoint":11.3,"signalPercentile":10},{"slug":"legal_bench","name":"LegalBench","family":"legal_bench","signalToNoise":2.213,"beta":0.59,"sigma":0.267,"communality":0.638,"difficulty":85.3,"nModels":136,"precision":17.0,"precisionMidpoint":11.4,"signalPercentile":10},{"slug":"astabench-2026-litqa-search","name":"LitQA2-FullText-Search (AstaBench 2026)","family":"astabench-2026-litqa-search","signalToNoise":2.195,"beta":0.938,"sigma":0.427,"communality":0.487,"difficulty":120.4,"nModels":13,"precision":16.3,"precisionMidpoint":11.5,"signalPercentile":10},{"slug":"hieroglyphbench","name":"HieroglyphBench","family":"hieroglyphbench","signalToNoise":2.178,"beta":1.153,"sigma":0.529,"communality":0.302,"difficulty":179.9,"nModels":30,"precision":11.7,"precisionMidpoint":11.6,"signalPercentile":10},{"slug":"cresta-bench","name":"Cresta-Bench","family":"cresta-bench","signalToNoise":2.175,"beta":1.344,"sigma":0.618,"communality":0.329,"difficulty":147.1,"nModels":14,"precision":13.5,"precisionMidpoint":11.6,"signalPercentile":10},{"slug":"elimination-game-trueskill-mu","name":"Elimination Game TrueSkill Mu","family":"elimination-game-trueskill-mu","signalToNoise":2.174,"beta":1.704,"sigma":0.784,"communality":0.378,"difficulty":137.9,"nModels":55,"precision":11.6,"precisionMidpoint":11.6,"signalPercentile":9},{"slug":"docvqa","name":"DocVQA","family":"docvqa","signalToNoise":2.17,"beta":0.715,"sigma":0.329,"communality":0.272,"difficulty":43.9,"nModels":58,"precision":37.5,"precisionMidpoint":11.6,"signalPercentile":9},{"slug":"edit-bench","name":"EDIT-Bench Complete","family":"edit-bench","signalToNoise":2.168,"beta":0.572,"sigma":0.264,"communality":0.434,"difficulty":137.5,"nModels":38,"precision":12.3,"precisionMidpoint":11.6,"signalPercentile":9},{"slug":"openeqa-em-eqa","name":"OpenEQA (episodic memory)","family":"openeqa-em-eqa","signalToNoise":2.16,"beta":0.794,"sigma":0.368,"communality":0.506,"difficulty":111.3,"nModels":8,"precision":16.7,"precisionMidpoint":11.7,"signalPercentile":9},{"slug":"vqav2-val","name":"VQAv2 (validation)","family":"vqav2-val","signalToNoise":2.117,"beta":0.634,"sigma":0.299,"communality":0.01,"difficulty":82.6,"nModels":10,"precision":19.5,"precisionMidpoint":11.9,"signalPercentile":9},{"slug":"gsm8k-openllm-v1-harness","name":"GSM8K (Open LLM Leaderboard v1)","family":"gsm8k-openllm-v1-harness","signalToNoise":2.088,"beta":1.4,"sigma":0.67,"communality":0.645,"difficulty":102.1,"nModels":91,"precision":39.0,"precisionMidpoint":12.1,"signalPercentile":9},{"slug":"snorkelspatial","name":"SnorkelSpatial","family":"snorkelspatial","signalToNoise":2.085,"beta":1.593,"sigma":0.764,"communality":0.403,"difficulty":147.3,"nModels":32,"precision":15.0,"precisionMidpoint":12.1,"signalPercentile":9},{"slug":"quac","name":"QuAC","family":"quac","signalToNoise":2.058,"beta":0.354,"sigma":0.172,"communality":0.783,"difficulty":115.5,"nModels":28,"precision":13.1,"precisionMidpoint":12.2,"signalPercentile":9},{"slug":"sage","name":"SAGE","family":"sage","signalToNoise":2.055,"beta":0.6,"sigma":0.292,"communality":0.469,"difficulty":167.1,"nModels":80,"precision":12.3,"precisionMidpoint":12.2,"signalPercentile":8},{"slug":"snorkel-sequences","name":"SnorkelSequences","family":"snorkel-sequences","signalToNoise":2.055,"beta":0.94,"sigma":0.457,"communality":0.453,"difficulty":130.0,"nModels":31,"precision":15.5,"precisionMidpoint":12.2,"signalPercentile":8},{"slug":"bigcodebench-complete","name":"BigCodeBench Complete","family":"bigcodebench-complete","signalToNoise":2.011,"beta":0.679,"sigma":0.337,"communality":0.575,"difficulty":118.8,"nModels":100,"precision":15.3,"precisionMidpoint":12.5,"signalPercentile":8},{"slug":"multipl-e-humaneval-8-language-average","name":"MultiPL-E (HumanEval, 8-language average)","family":"multipl-e-humaneval-8-language-average","signalToNoise":2.002,"beta":0.902,"sigma":0.451,"communality":0.637,"difficulty":102.4,"nModels":84,"precision":22.6,"precisionMidpoint":12.6,"signalPercentile":8},{"slug":"cv-bench","name":"CV-Bench","family":"cv-bench","signalToNoise":1.986,"beta":0.595,"sigma":0.3,"communality":0.0,"difficulty":90.8,"nModels":8,"precision":25.3,"precisionMidpoint":12.7,"signalPercentile":8},{"slug":"openai-mrcr-2-needle-128k","name":"OpenAI-MRCR 2-needle - 128K","family":"openai-mrcr-2-needle-128k","signalToNoise":1.978,"beta":1.41,"sigma":0.713,"communality":0.328,"difficulty":138.7,"nModels":13,"precision":17.3,"precisionMidpoint":12.7,"signalPercentile":8},{"slug":"pope-f1","name":"POPE (F1, pooled over three settings)","family":"pope-f1","signalToNoise":1.977,"beta":0.506,"sigma":0.256,"communality":0.0,"difficulty":22.5,"nModels":15,"precision":29.2,"precisionMidpoint":12.7,"signalPercentile":8},{"slug":"embspatial-bench","name":"EmbSpatial-Bench","family":"embspatial-bench","signalToNoise":1.973,"beta":0.563,"sigma":0.285,"communality":0.416,"difficulty":100.6,"nModels":29,"precision":19.8,"precisionMidpoint":12.8,"signalPercentile":7},{"slug":"cyberbench-v1-1-overall","name":"CyberBench v1.1 Overall","family":"cyberbench-v1-1-overall","signalToNoise":1.957,"beta":0.891,"sigma":0.455,"communality":0.136,"difficulty":138.2,"nModels":36,"precision":14.7,"precisionMidpoint":12.9,"signalPercentile":7},{"slug":"tau2-bench-telecom","name":"\u03c4\u00b2-bench (Telecom)","family":"tau2-bench-telecom","signalToNoise":1.952,"beta":1.66,"sigma":0.85,"communality":0.337,"difficulty":133.8,"nModels":325,"precision":21.9,"precisionMidpoint":12.9,"signalPercentile":7},{"slug":"medic-summarization","name":"MEDIC (clinical summarization)","family":"medic-summarization","signalToNoise":1.899,"beta":0.368,"sigma":0.194,"communality":0.188,"difficulty":5.0,"nModels":49,"precision":23.9,"precisionMidpoint":13.3,"signalPercentile":7},{"slug":"refcoco-8-split-average","name":"RefCOCO, RefCOCO+ and RefCOCOg eight-split average","family":"refcoco-8-split-average","signalToNoise":1.867,"beta":0.754,"sigma":0.404,"communality":0.0,"difficulty":77.5,"nModels":8,"precision":28.0,"precisionMidpoint":13.5,"signalPercentile":7},{"slug":"ruby-llm-program-fixer","name":"Ruby LLM Benchmarks \u2014 Program Fixer","family":"ruby-llm-program-fixer","signalToNoise":1.849,"beta":0.469,"sigma":0.254,"communality":0.402,"difficulty":113.4,"nModels":111,"precision":15.4,"precisionMidpoint":13.6,"signalPercentile":7},{"slug":"mvbench","name":"MVBench","family":"mvbench","signalToNoise":1.844,"beta":0.445,"sigma":0.241,"communality":0.472,"difficulty":89.9,"nModels":47,"precision":17.3,"precisionMidpoint":13.7,"signalPercentile":7},{"slug":"hr-bench-8k","name":"HR-Bench (8K)","family":"hr-bench-8k","signalToNoise":1.829,"beta":0.602,"sigma":0.329,"communality":0.193,"difficulty":116.1,"nModels":8,"precision":20.0,"precisionMidpoint":13.8,"signalPercentile":7},{"slug":"cc-ocr","name":"CC-OCR","family":"cc-ocr","signalToNoise":1.815,"beta":0.484,"sigma":0.267,"communality":0.36,"difficulty":81.4,"nModels":41,"precision":19.7,"precisionMidpoint":13.9,"signalPercentile":6},{"slug":"ccpm","name":"CCPM","family":"ccpm","signalToNoise":1.813,"beta":0.849,"sigma":0.468,"communality":0.061,"difficulty":76.9,"nModels":11,"precision":39.4,"precisionMidpoint":13.9,"signalPercentile":6},{"slug":"medic-open-ended-elo","name":"MEDIC (open-ended Elo)","family":"medic-open-ended-elo","signalToNoise":1.803,"beta":1.16,"sigma":0.644,"communality":0.594,"difficulty":118.5,"nModels":48,"precision":14.0,"precisionMidpoint":14.0,"signalPercentile":6},{"slug":"astabench-2026-scholarqa-cs2","name":"ScholarQA-CS2 (AstaBench 2026)","family":"astabench-2026-scholarqa-cs2","signalToNoise":1.774,"beta":0.956,"sigma":0.539,"communality":0.38,"difficulty":120.3,"nModels":13,"precision":20.5,"precisionMidpoint":14.2,"signalPercentile":6},{"slug":"regretbench-success","name":"RegretBench (goal success)","family":"regretbench-success","signalToNoise":1.754,"beta":0.72,"sigma":0.411,"communality":0.061,"difficulty":101.6,"nModels":12,"precision":21.4,"precisionMidpoint":14.4,"signalPercentile":6},{"slug":"arabic-scale-ai-2","name":"Arabic","family":"arabic-scale-ai-2","signalToNoise":1.751,"beta":1.34,"sigma":0.765,"communality":0.375,"difficulty":128.2,"nModels":18,"precision":14.4,"precisionMidpoint":14.4,"signalPercentile":6},{"slug":"lingoqa","name":"LingoQA","family":"lingoqa","signalToNoise":1.732,"beta":1.135,"sigma":0.655,"communality":0.23,"difficulty":131.3,"nModels":21,"precision":19.9,"precisionMidpoint":14.5,"signalPercentile":6},{"slug":"p2npcbench","name":"P2NPCBench (NP-complete, pass@1)","family":"p2npcbench","signalToNoise":1.725,"beta":0.872,"sigma":0.506,"communality":0.586,"difficulty":163.6,"nModels":11,"precision":14.6,"precisionMidpoint":14.6,"signalPercentile":5},{"slug":"mortgage_tax-vals-all-1258","name":"MortgageTax (Vals all 1,258 documents)","family":"mortgage_tax-vals-all-1258","signalToNoise":1.717,"beta":0.514,"sigma":0.299,"communality":0.453,"difficulty":124.7,"nModels":85,"precision":16.3,"precisionMidpoint":14.7,"signalPercentile":5},{"slug":"mbpp-plus","name":"MBPP+","family":"mbpp-plus","signalToNoise":1.715,"beta":0.557,"sigma":0.325,"communality":0.6,"difficulty":89.4,"nModels":104,"precision":20.7,"precisionMidpoint":14.7,"signalPercentile":5},{"slug":"facts-grounding","name":"FACTS Grounding","family":"facts-grounding","signalToNoise":1.698,"beta":0.671,"sigma":0.395,"communality":0.0,"difficulty":80.0,"nModels":19,"precision":26.3,"precisionMidpoint":14.8,"signalPercentile":5},{"slug":"robospatial-home","name":"RoboSpatial-Home","family":"robospatial-home","signalToNoise":1.688,"beta":0.514,"sigma":0.305,"communality":0.459,"difficulty":121.1,"nModels":10,"precision":16.8,"precisionMidpoint":14.9,"signalPercentile":5},{"slug":"healthbench-card","name":"HealthBench","family":"healthbench-card","signalToNoise":1.686,"beta":0.425,"sigma":0.252,"communality":0.379,"difficulty":138.6,"nModels":39,"precision":15.4,"precisionMidpoint":14.9,"signalPercentile":5},{"slug":"lmarena-search-arena","name":"Search Arena","family":"lmarena-search-arena","signalToNoise":1.686,"beta":1.345,"sigma":0.798,"communality":0.292,"difficulty":158.3,"nModels":17,"precision":14.9,"precisionMidpoint":14.9,"signalPercentile":5},{"slug":"chessbench-ai-coherence","name":"ChessBench Coherence","family":"chessbench-ai-coherence","signalToNoise":1.681,"beta":1.212,"sigma":0.721,"communality":0.378,"difficulty":142.3,"nModels":65,"precision":19.6,"precisionMidpoint":15.0,"signalPercentile":5},{"slug":"binaryaudit","name":"BinaryAudit (Binary Analysis)","family":"binaryaudit","signalToNoise":1.667,"beta":1.134,"sigma":0.68,"communality":0.026,"difficulty":182.1,"nModels":20,"precision":15.9,"precisionMidpoint":15.1,"signalPercentile":4},{"slug":"ocrbench","name":"OCRBench","family":"ocrbench","signalToNoise":1.661,"beta":0.439,"sigma":0.264,"communality":0.302,"difficulty":37.3,"nModels":73,"precision":26.8,"precisionMidpoint":15.2,"signalPercentile":4},{"slug":"humaneval-plus","name":"HumanEval+","family":"humaneval-plus","signalToNoise":1.646,"beta":0.949,"sigma":0.576,"communality":0.538,"difficulty":100.7,"nModels":112,"precision":29.7,"precisionMidpoint":15.3,"signalPercentile":4},{"slug":"humaneval-mul","name":"HumanEval-Mul","family":"humaneval-mul","signalToNoise":1.626,"beta":0.635,"sigma":0.391,"communality":0.697,"difficulty":84.1,"nModels":18,"precision":25.1,"precisionMidpoint":15.5,"signalPercentile":4},{"slug":"slop-score","name":"Slop Score","family":"slop-score","signalToNoise":1.622,"beta":1.011,"sigma":0.623,"communality":0.354,"difficulty":133.1,"nModels":19,"precision":19.5,"precisionMidpoint":15.5,"signalPercentile":4},{"slug":"pubmedqa-openmedical-harness","name":"PubMedQA (Open Medical-LLM)","family":"pubmedqa-openmedical-harness","signalToNoise":1.61,"beta":0.475,"sigma":0.295,"communality":0.386,"difficulty":85.7,"nModels":17,"precision":26.2,"precisionMidpoint":15.6,"signalPercentile":4},{"slug":"where2place","name":"Where2Place","family":"where2place","signalToNoise":1.598,"beta":0.707,"sigma":0.443,"communality":0.335,"difficulty":150.2,"nModels":24,"precision":16.3,"precisionMidpoint":15.8,"signalPercentile":4},{"slug":"race-high","name":"RACE-High","family":"race-high","signalToNoise":1.559,"beta":0.85,"sigma":0.545,"communality":0.404,"difficulty":108.2,"nModels":33,"precision":29.6,"precisionMidpoint":16.1,"signalPercentile":3},{"slug":"musr-openllm-harness","name":"MuSR (Open LLM Leaderboard v2)","family":"musr-openllm-harness","signalToNoise":1.547,"beta":0.187,"sigma":0.121,"communality":0.472,"difficulty":151.1,"nModels":121,"precision":16.3,"precisionMidpoint":16.3,"signalPercentile":3},{"slug":"txbench-od-v1-113","name":"TxBench Oligo Discovery V1 (113 evaluations)","family":"txbench-od-v1-113","signalToNoise":1.533,"beta":0.898,"sigma":0.586,"communality":0.0,"difficulty":180.8,"nModels":24,"precision":16.8,"precisionMidpoint":16.4,"signalPercentile":3},{"slug":"geobench","name":"GeoBench","family":"geobench","signalToNoise":1.517,"beta":0.658,"sigma":0.433,"communality":0.316,"difficulty":107.8,"nModels":38,"precision":22.2,"precisionMidpoint":16.6,"signalPercentile":3},{"slug":"tau3-bench","name":"\u03c4\u00b3-bench average","family":"tau3-bench","signalToNoise":1.486,"beta":0.362,"sigma":0.244,"communality":0.0,"difficulty":102.3,"nModels":20,"precision":18.9,"precisionMidpoint":16.9,"signalPercentile":3},{"slug":"infovqa","name":"InfographicVQA","family":"infovqa","signalToNoise":1.472,"beta":0.624,"sigma":0.424,"communality":0.518,"difficulty":82.1,"nModels":37,"precision":30.4,"precisionMidpoint":17.1,"signalPercentile":3},{"slug":"cyberbench-overall","name":"CyberBench Overall","family":"cyberbench-overall","signalToNoise":1.451,"beta":0.841,"sigma":0.58,"communality":0.001,"difficulty":132.8,"nModels":19,"precision":20.5,"precisionMidpoint":17.4,"signalPercentile":3},{"slug":"deepresearchbench-ii-mercor-130","name":"DeepResearch Bench II","family":"deepresearchbench-ii","signalToNoise":1.427,"beta":0.562,"sigma":0.394,"communality":0.0,"difficulty":170.1,"nModels":25,"precision":17.6,"precisionMidpoint":17.6,"signalPercentile":3},{"slug":"afantasia-valid-retry","name":"A-Fantasia (Valid Attempts with Retries)","family":"afantasia-valid-retry","signalToNoise":1.421,"beta":0.764,"sigma":0.538,"communality":0.184,"difficulty":150.9,"nModels":56,"precision":18.4,"precisionMidpoint":17.7,"signalPercentile":2},{"slug":"vstar-bench","name":"V* Bench","family":"vstar-bench","signalToNoise":1.389,"beta":0.725,"sigma":0.522,"communality":0.107,"difficulty":91.7,"nModels":25,"precision":30.5,"precisionMidpoint":18.1,"signalPercentile":2},{"slug":"chartqa-test","name":"ChartQA (test)","family":"chartqa-test","signalToNoise":1.384,"beta":0.592,"sigma":0.428,"communality":0.2,"difficulty":63.4,"nModels":65,"precision":36.3,"precisionMidpoint":18.2,"signalPercentile":2},{"slug":"pgg-bench-trueskill-mu","name":"PGG-Bench TrueSkill Mu","family":"pgg-bench-trueskill-mu","signalToNoise":1.374,"beta":1.184,"sigma":0.862,"communality":0.137,"difficulty":134.4,"nModels":17,"precision":18.3,"precisionMidpoint":18.3,"signalPercentile":2},{"slug":"snakebench-rating","name":"SnakeBench Rating","family":"snakebench-rating","signalToNoise":1.367,"beta":1.031,"sigma":0.754,"communality":0.432,"difficulty":134.9,"nModels":205,"precision":18.4,"precisionMidpoint":18.4,"signalPercentile":2},{"slug":"infinitebench-en-qa","name":"InfiniteBench En.QA","family":"infinitebench-en-qa","signalToNoise":1.337,"beta":0.61,"sigma":0.457,"communality":0.0,"difficulty":170.0,"nModels":8,"precision":18.8,"precisionMidpoint":18.8,"signalPercentile":2},{"slug":"chid","name":"ChID","family":"chid","signalToNoise":1.314,"beta":0.861,"sigma":0.655,"communality":0.0,"difficulty":63.6,"nModels":8,"precision":59.0,"precisionMidpoint":19.2,"signalPercentile":2},{"slug":"blxbench","name":"BLXBench","family":"blxbench","signalToNoise":1.312,"beta":0.433,"sigma":0.33,"communality":0.0,"difficulty":103.4,"nModels":29,"precision":22.2,"precisionMidpoint":19.2,"signalPercentile":1},{"slug":"ruler-128k","name":"RULER - 128K","family":"ruler-128k","signalToNoise":1.284,"beta":1.178,"sigma":0.917,"communality":0.108,"difficulty":105.1,"nModels":26,"precision":44.9,"precisionMidpoint":19.6,"signalPercentile":1},{"slug":"screenspot-pro","name":"ScreenSpot-Pro","family":"screenspot-pro","signalToNoise":1.274,"beta":1.066,"sigma":0.837,"communality":0.351,"difficulty":142.6,"nModels":47,"precision":22.9,"precisionMidpoint":19.8,"signalPercentile":1},{"slug":"ifeval-openllm-harness","name":"IFEval (Open LLM Leaderboard v2)","family":"ifeval-openllm-harness","signalToNoise":1.261,"beta":0.834,"sigma":0.661,"communality":0.369,"difficulty":105.3,"nModels":121,"precision":31.9,"precisionMidpoint":20.0,"signalPercentile":1},{"slug":"multipl-e-humaneval-csharp","name":"MultiPL-E HumanEval (C#)","family":"multipl-e-humaneval-csharp","signalToNoise":1.145,"beta":0.784,"sigma":0.685,"communality":0.403,"difficulty":98.7,"nModels":51,"precision":36.3,"precisionMidpoint":22.0,"signalPercentile":1},{"slug":"point-bench","name":"Point-Bench","family":"point-bench","signalToNoise":1.109,"beta":0.691,"sigma":0.623,"communality":0.1,"difficulty":123.2,"nModels":8,"precision":27.2,"precisionMidpoint":22.7,"signalPercentile":1},{"slug":"caselaw-v2","name":"CaseLaw","family":"caselaw-v2","signalToNoise":1.089,"beta":0.323,"sigma":0.297,"communality":0.007,"difficulty":116.8,"nModels":52,"precision":24.4,"precisionMidpoint":23.1,"signalPercentile":1},{"slug":"medic-closed-ended","name":"MEDIC (closed-ended)","family":"medic-closed-ended","signalToNoise":1.073,"beta":0.501,"sigma":0.467,"communality":0.205,"difficulty":95.1,"nModels":43,"precision":30.0,"precisionMidpoint":23.5,"signalPercentile":1},{"slug":"vsi-bench","name":"VSI-Bench","family":"vsi-bench","signalToNoise":1.049,"beta":0.554,"sigma":0.528,"communality":0.024,"difficulty":147.8,"nModels":8,"precision":24.6,"precisionMidpoint":24.0,"signalPercentile":0},{"slug":"visual-acuity-4k-threshold-row","name":"Visual Acuity 4K Threshold Row","family":"visual-acuity-4k-threshold-row","signalToNoise":0.929,"beta":0.578,"sigma":0.622,"communality":0.044,"difficulty":119.9,"nModels":29,"precision":31.3,"precisionMidpoint":27.1,"signalPercentile":0},{"slug":"hardgeobench-country-accuracy","name":"HardGeoBench Avg Distance","family":"hardgeobench-avg-distance-miles","signalToNoise":0.891,"beta":0.404,"sigma":0.454,"communality":0.077,"difficulty":153.5,"nModels":70,"precision":28.5,"precisionMidpoint":28.3,"signalPercentile":0},{"slug":"refspatial-bench","name":"RefSpatial-Bench","family":"refspatial-bench","signalToNoise":0.808,"beta":0.703,"sigma":0.87,"communality":0.108,"difficulty":156.2,"nModels":22,"precision":31.6,"precisionMidpoint":31.1,"signalPercentile":0}],"models":[{"slug":"gpt-6-astra-pro","modelId":"gpt-6-astra-pro","modelSlug":"gpt-6-astra-pro","name":"GPT-6 Astra Pro","organizationSlug":"openai","n":6,"rankEligible":true,"score":170.07,"scoreLo":164.8,"scoreHi":176.7,"modelScoreStateId":"gpt-6-astra-pro::effort:max","modelScoreConfigSlug":"gpt-6-astra-pro-max","modelScoreStateName":"GPT-6 Astra Pro (max)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":1.7764,"rankLo":1.0,"rankHi":7.0},{"slug":"claude-opus-5-5","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5","organizationSlug":"anthropic","n":125,"rankEligible":true,"score":168.71,"scoreLo":164.52,"scoreHi":173.74,"modelScoreStateId":"claude-opus-5-5::effort:max","modelScoreConfigSlug":"claude-opus-5-5-max","modelScoreStateName":"Claude Opus 5.5 (max)","modelScoreStateBestProbability":0.9469444444444445,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":2.2569,"rankLo":1.0,"rankHi":4.0},{"slug":"gemini-4-carbon","modelId":"gemini-4-carbon","modelSlug":"gemini-4-carbon","name":"Gemini 4 Carbon","organizationSlug":"google","n":27,"rankEligible":true,"score":168.62,"scoreLo":164.37,"scoreHi":173.84,"modelScoreStateId":"gemini-4-carbon::effort:high","modelScoreConfigSlug":"gemini-4-carbon-high","modelScoreStateName":"Gemini 4 Carbon (high)","modelScoreStateBestProbability":0.835,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":2.6239,"rankLo":1.0,"rankHi":4.0},{"slug":"gpt-6-astra","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra","organizationSlug":"openai","n":140,"rankEligible":true,"score":168.24,"scoreLo":164.1,"scoreHi":173.18,"modelScoreStateId":"gpt-6-astra::effort:max","modelScoreConfigSlug":"gpt-6-astra-max","modelScoreStateName":"GPT-6 Astra (max)","modelScoreStateBestProbability":0.96,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":3.6394,"rankLo":2.0,"rankHi":5.0},{"slug":"claude-sonnet-5-5","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","organizationSlug":"anthropic","n":99,"rankEligible":true,"score":167.5,"scoreLo":163.5,"scoreHi":172.37,"modelScoreStateId":"claude-sonnet-5-5::effort:max","modelScoreConfigSlug":"claude-sonnet-5-5-max","modelScoreStateName":"Claude Sonnet 5.5 (max)","modelScoreStateBestProbability":0.9627777777777777,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":5.3219,"rankLo":4.0,"rankHi":7.0},{"slug":"gpt-6-1-sol","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol","organizationSlug":"openai","n":119,"rankEligible":true,"score":167.31,"scoreLo":163.26,"scoreHi":172.09,"modelScoreStateId":"gpt-6-1-sol::effort:max","modelScoreConfigSlug":"gpt-6-1-sol-max","modelScoreStateName":"GPT-6.1 Sol (max)","modelScoreStateBestProbability":0.9258333333333333,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":5.9264,"rankLo":5.0,"rankHi":8.0},{"slug":"gemini-4-argon","modelId":"gemini-4-argon","modelSlug":"gemini-4-argon","name":"Gemini 4 Argon","organizationSlug":"google","n":140,"rankEligible":true,"score":167.01,"scoreLo":163.04,"scoreHi":171.73,"modelScoreStateId":"gemini-4-argon::effort:high","modelScoreConfigSlug":"gemini-4-argon-high","modelScoreStateName":"Gemini 4 Argon (high)","modelScoreStateBestProbability":0.8283333333333334,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":7.1197,"rankLo":5.0,"rankHi":9.0},{"slug":"claude-fable-5-1","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1","organizationSlug":"anthropic","n":121,"rankEligible":true,"score":166.81,"scoreLo":162.9,"scoreHi":171.55,"modelScoreStateId":"claude-fable-5-1::effort:max","modelScoreConfigSlug":"claude-fable-5-1-max","modelScoreStateName":"Claude Fable 5.1 (max)","modelScoreStateBestProbability":0.9683333333333334,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":7.8703,"rankLo":6.0,"rankHi":9.0},{"slug":"gpt-6-1-terra","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra","organizationSlug":"openai","n":36,"rankEligible":true,"score":166.21,"scoreLo":162.21,"scoreHi":170.99,"modelScoreStateId":"gpt-6-1-terra::effort:max","modelScoreConfigSlug":"gpt-6-1-terra-max","modelScoreStateName":"GPT-6.1 Terra (max)","modelScoreStateBestProbability":0.935,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":9.4775,"rankLo":7.0,"rankHi":12.0},{"slug":"muse-wonder-full","modelId":"muse-wonder-full","modelSlug":"muse-wonder-full","name":"Muse Wonder Full","organizationSlug":"meta","n":45,"rankEligible":true,"score":165.73,"scoreLo":161.91,"scoreHi":170.28,"modelScoreStateId":"muse-wonder-full::effort:max","modelScoreConfigSlug":"muse-wonder-full-max","modelScoreStateName":"Muse Wonder Full (max)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":10.9239,"rankLo":9.0,"rankHi":14.0},{"slug":"kimi-k3-1-pro","modelId":"kimi-k3-1-pro","modelSlug":"kimi-k3-1-pro","name":"Kimi K3.1 Pro","organizationSlug":"moonshot-ai","n":38,"rankEligible":true,"score":165.66,"scoreLo":161.76,"scoreHi":170.27,"modelScoreStateId":"kimi-k3-1-pro::effort:max","modelScoreConfigSlug":"kimi-k3-1-pro-max","modelScoreStateName":"Kimi K3.1 Pro (max)","modelScoreStateBestProbability":0.7936111111111112,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":11.2256,"rankLo":9.0,"rankHi":14.0},{"slug":"claude-fable-5","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5","organizationSlug":"anthropic","n":36,"rankEligible":true,"score":165.42,"scoreLo":161.48,"scoreHi":170.2,"modelScoreStateId":"claude-fable-5::enabled","modelScoreConfigSlug":"claude-fable-5","modelScoreStateName":"Claude Fable 5 (enabled)","modelScoreStateBestProbability":0.9175,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":12.2692,"rankLo":9.0,"rankHi":16.0},{"slug":"claude-opus-5","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5","organizationSlug":"anthropic","n":142,"rankEligible":true,"score":165.2,"scoreLo":161.43,"scoreHi":169.68,"modelScoreStateId":"claude-opus-5::effort:max","modelScoreConfigSlug":"claude-opus-5-max","modelScoreStateName":"Claude Opus 5 (max)","modelScoreStateBestProbability":0.9,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":13.1725,"rankLo":11.0,"rankHi":16.0},{"slug":"gpt-6-sol","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol","organizationSlug":"openai","n":103,"rankEligible":true,"score":164.9,"scoreLo":161.22,"scoreHi":169.4,"modelScoreStateId":"gpt-6-sol::effort:max","modelScoreConfigSlug":"gpt-6-sol-max","modelScoreStateName":"GPT-6 Sol (max)","modelScoreStateBestProbability":0.9963888888888889,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":14.5261,"rankLo":12.0,"rankHi":17.0},{"slug":"gpt-5-6-sol-pro","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro","organizationSlug":"openai","n":30,"rankEligible":true,"score":164.63,"scoreLo":160.76,"scoreHi":169.36,"modelScoreStateId":"gpt-5-6-sol-pro::effort:max","modelScoreConfigSlug":"gpt-5-6-sol-pro-max","modelScoreStateName":"GPT-5.6 Sol Pro (max)","modelScoreStateBestProbability":0.9997222222222222,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":15.4581,"rankLo":10.0,"rankHi":20.0},{"slug":"muse-wonder","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder","organizationSlug":"meta","n":77,"rankEligible":true,"score":164.54,"scoreLo":160.81,"scoreHi":168.94,"modelScoreStateId":"muse-wonder::effort:max","modelScoreConfigSlug":"muse-wonder-max","modelScoreStateName":"Muse Wonder (max)","modelScoreStateBestProbability":0.8002777777777778,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":15.9939,"rankLo":14.0,"rankHi":18.0},{"slug":"gpt-5-6-sol","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol","organizationSlug":"openai","n":156,"rankEligible":true,"score":163.97,"scoreLo":160.37,"scoreHi":168.37,"modelScoreStateId":"gpt-5-6-sol::effort:max","modelScoreConfigSlug":"gpt-5-6-sol-max","modelScoreStateName":"GPT-5.6 Sol (max)","modelScoreStateBestProbability":0.9880555555555556,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":18.0175,"rankLo":16.0,"rankHi":20.0},{"slug":"claude-mythos-preview","modelId":"claude-mythos-preview","modelSlug":"claude-mythos-preview","name":"Claude Mythos Preview","organizationSlug":"anthropic","n":5,"rankEligible":true,"score":163.7,"scoreLo":159.08,"scoreHi":168.87,"modelScoreStateId":"claude-mythos-preview::enabled","modelScoreConfigSlug":"claude-mythos-preview-caisi-math-prompt-no-system-safeguards","modelScoreStateName":"Claude Mythos Preview (enabled)","modelScoreStateBestProbability":0.5383333333333333,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":19.8339,"rankLo":9.0,"rankHi":40.0},{"slug":"gpt-5-5-pro","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro","organizationSlug":"openai","n":18,"rankEligible":true,"score":163.33,"scoreLo":159.58,"scoreHi":167.83,"modelScoreStateId":"gpt-5-5-pro::effort:xhigh","modelScoreConfigSlug":"gpt-5-5-pro-xhigh","modelScoreStateName":"GPT-5.5 Pro (xhigh)","modelScoreStateBestProbability":0.9947222222222222,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":20.2056,"rankLo":16.0,"rankHi":27.0},{"slug":"longcat-2-5","modelId":"longcat-2-5","modelSlug":"longcat-2-5","name":"LongCat-2.5","organizationSlug":"meituan","n":13,"rankEligible":true,"score":163.25,"scoreLo":159.51,"scoreHi":167.7,"modelScoreStateId":"longcat-2-5::enabled","modelScoreConfigSlug":"longcat-2-5-thinking","modelScoreStateName":"LongCat-2.5 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":20.7269,"rankLo":15.0,"rankHi":31.0},{"slug":"deepseek-v4-1-flash-1017","modelId":"deepseek-v4-1-flash-1017","modelSlug":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017","organizationSlug":"deepseek","n":16,"rankEligible":true,"score":162.61,"scoreLo":159.08,"scoreHi":166.85,"modelScoreStateId":"deepseek-v4-1-flash-1017::default","modelScoreConfigSlug":"deepseek-v4-1-flash-1017","modelScoreStateName":"DeepSeek V4.1 Flash 1017","modelScoreStateBestProbability":0.6891666666666667,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":23.315,"rankLo":19.0,"rankHi":31.0},{"slug":"claude-haiku-5-5","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5","organizationSlug":"anthropic","n":13,"rankEligible":true,"score":162.59,"scoreLo":159.09,"scoreHi":166.72,"modelScoreStateId":"claude-haiku-5-5::default","modelScoreConfigSlug":"claude-haiku-5-5","modelScoreStateName":"Claude Haiku 5.5","modelScoreStateBestProbability":0.6741666666666667,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":23.2561,"rankLo":19.0,"rankHi":29.0},{"slug":"grok-4-7","modelId":"grok-4-7","modelSlug":"grok-4-7","name":"Grok 4.7","organizationSlug":"xai","n":44,"rankEligible":true,"score":162.43,"scoreLo":158.84,"scoreHi":166.57,"modelScoreStateId":"grok-4-7::default","modelScoreConfigSlug":"grok-4-7","modelScoreStateName":"Grok 4.7","modelScoreStateBestProbability":0.8966666666666666,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":24.3531,"rankLo":20.0,"rankHi":31.0},{"slug":"gpt-5-6-terra-pro","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro","organizationSlug":"openai","n":5,"rankEligible":true,"score":162.36,"scoreLo":158.03,"scoreHi":167.26,"modelScoreStateId":"gpt-5-6-terra-pro::default","modelScoreConfigSlug":"gpt-5-6-terra-pro","modelScoreStateName":"GPT-5.6 Terra Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":27.2925,"rankLo":15.0,"rankHi":51.0},{"slug":"macaron-v1-venti","modelId":"macaron-v1-venti","modelSlug":"macaron-v1-venti","name":"Macaron-V1-Venti","organizationSlug":"mindlab-research","n":4,"rankEligible":true,"score":162.23,"scoreLo":156.01,"scoreHi":169.91,"modelScoreStateId":"macaron-v1-venti::default","modelScoreConfigSlug":"macaron-v1-venti-mol-router","modelScoreStateName":"Macaron-V1-Venti","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":32.4064,"rankLo":4.0,"rankHi":73.0},{"slug":"muse-spark-1-3","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3","organizationSlug":"meta","n":83,"rankEligible":true,"score":162.13,"scoreLo":158.71,"scoreHi":166.28,"modelScoreStateId":"muse-spark-1-3::effort:max","modelScoreConfigSlug":"muse-spark-1-3-max","modelScoreStateName":"Muse Spark 1.3 (max)","modelScoreStateBestProbability":0.9963888888888889,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":26.5578,"rankLo":22.0,"rankHi":32.0},{"slug":"gpt-5-6-terra","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra","organizationSlug":"openai","n":118,"rankEligible":true,"score":161.87,"scoreLo":158.43,"scoreHi":165.96,"modelScoreStateId":"gpt-5-6-terra::effort:max","modelScoreConfigSlug":"gpt-5-6-terra-max","modelScoreStateName":"GPT-5.6 Terra (max)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":29.0,"rankLo":24.0,"rankHi":35.0},{"slug":"gemini-3-deep-think","modelId":"gemini-3-deep-think","modelSlug":"gemini-3-deep-think","name":"Gemini 3 Deep Think","organizationSlug":"google","n":6,"rankEligible":true,"score":161.79,"scoreLo":157.75,"scoreHi":166.61,"modelScoreStateId":"gemini-3-deep-think::default","modelScoreConfigSlug":"gemini-3-deep-think","modelScoreStateName":"Gemini 3 Deep Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":31.2811,"rankLo":18.0,"rankHi":53.0},{"slug":"gemini-3-8-flash","modelId":"gemini-3-8-flash","modelSlug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","organizationSlug":"google","n":38,"rankEligible":true,"score":161.78,"scoreLo":158.27,"scoreHi":165.87,"modelScoreStateId":"gemini-3-8-flash::effort:medium","modelScoreConfigSlug":"gemini-3-8-flash-medium","modelScoreStateName":"Gemini 3.8 Flash (medium)","modelScoreStateBestProbability":0.9358333333333333,"modelScoreReasoningCondition":"effort:medium","modelScoreReasoningEffort":"medium","modelScoreHarness":"none","rankMean":29.9931,"rankLo":23.0,"rankHi":38.0},{"slug":"gpt-5-4-pro","modelId":"gpt-5-4-pro","modelSlug":"gpt-5-4-pro","name":"GPT-5.4 Pro","organizationSlug":"openai","n":17,"rankEligible":true,"score":161.77,"scoreLo":158.25,"scoreHi":166.11,"modelScoreStateId":"gpt-5-4-pro::effort:xhigh","modelScoreConfigSlug":"gpt-5-4-pro-xhigh","modelScoreStateName":"GPT-5.4 Pro (xhigh)","modelScoreStateBestProbability":0.9938888888888889,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":30.3794,"rankLo":21.0,"rankHi":43.0},{"slug":"grok-4-6","modelId":"grok-4-6","modelSlug":"grok-4-6","name":"Grok 4.6","organizationSlug":"xai","n":62,"rankEligible":true,"score":161.77,"scoreLo":158.34,"scoreHi":165.83,"modelScoreStateId":"grok-4-6::default","modelScoreConfigSlug":"grok-4-6","modelScoreStateName":"Grok 4.6","modelScoreStateBestProbability":0.9127777777777778,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":29.9567,"rankLo":24.0,"rankHi":37.0},{"slug":"mimo-v2-6-pro","modelId":"mimo-v2-6-pro","modelSlug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","organizationSlug":"xiaomi","n":17,"rankEligible":true,"score":161.59,"scoreLo":158.11,"scoreHi":165.88,"modelScoreStateId":"mimo-v2-6-pro::enabled","modelScoreConfigSlug":"mimo-v2-6-pro-thinking","modelScoreStateName":"MiMo-V2.6-Pro (thinking)","modelScoreStateBestProbability":0.7738888888888888,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":32.1519,"rankLo":22.0,"rankHi":45.0},{"slug":"qwen3-8-max-0902","modelId":"qwen3-8-max-0902","modelSlug":"qwen3-8-max-0902","name":"Qwen3.8 Max (0902)","organizationSlug":"alibaba","n":21,"rankEligible":true,"score":161.44,"scoreLo":158.05,"scoreHi":165.64,"modelScoreStateId":"qwen3-8-max-0902::default","modelScoreConfigSlug":"qwen3-8-max-0902","modelScoreStateName":"Qwen3.8 Max (0902)","modelScoreStateBestProbability":0.5105555555555555,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":33.5647,"rankLo":24.0,"rankHi":45.0},{"slug":"deepseek-v4-1-flash","modelId":"deepseek-v4-1-flash","modelSlug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","organizationSlug":"deepseek","n":49,"rankEligible":true,"score":161.25,"scoreLo":157.91,"scoreHi":165.3,"modelScoreStateId":"deepseek-v4-1-flash::default","modelScoreConfigSlug":"deepseek-v4-1-flash","modelScoreStateName":"DeepSeek V4.1 Flash","modelScoreStateBestProbability":0.9438888888888889,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":35.3808,"rankLo":28.0,"rankHi":43.025},{"slug":"gpt-5-5","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5","organizationSlug":"openai","n":182,"rankEligible":true,"score":161.16,"scoreLo":157.82,"scoreHi":165.13,"modelScoreStateId":"gpt-5-5::effort:xhigh","modelScoreConfigSlug":"gpt-5-5-xhigh","modelScoreStateName":"GPT-5.5 (xhigh)","modelScoreStateBestProbability":0.8897222222222222,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":36.3083,"rankLo":30.0,"rankHi":42.0},{"slug":"ling-3-1-flash-1019","modelId":"ling-3-1-flash-1019","modelSlug":"ling-3-1-flash-1019","name":"Ling 3.1 Flash 1019","organizationSlug":"inclusion-ai","n":10,"rankEligible":true,"score":161.05,"scoreLo":157.5,"scoreHi":165.25,"modelScoreStateId":"ling-3-1-flash-1019::enabled","modelScoreConfigSlug":"ling-3-1-flash-1019-thinking","modelScoreStateName":"Ling 3.1 Flash 1019 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":37.7272,"rankLo":25.0,"rankHi":52.0},{"slug":"minimax-m3-1-flash","modelId":"minimax-m3-1-flash","modelSlug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","organizationSlug":"minimax","n":19,"rankEligible":true,"score":160.97,"scoreLo":157.4,"scoreHi":165.33,"modelScoreStateId":"minimax-m3-1-flash::effort:max","modelScoreConfigSlug":"minimax-m3-1-flash-max","modelScoreStateName":"MiniMax M3.1 Flash (max)","modelScoreStateBestProbability":0.59,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":38.5614,"rankLo":24.0,"rankHi":54.0},{"slug":"qwen3-8-max","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max","organizationSlug":"alibaba","n":16,"rankEligible":true,"score":160.9,"scoreLo":157.34,"scoreHi":165.08,"modelScoreStateId":"qwen3-8-max::enabled","modelScoreConfigSlug":"qwen3-8-max-thinking","modelScoreStateName":"Qwen3.8-Max (thinking)","modelScoreStateBestProbability":0.8561111111111112,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":39.2617,"rankLo":25.0,"rankHi":53.0},{"slug":"claude-opus-4-8","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8","organizationSlug":"anthropic","n":146,"rankEligible":true,"score":160.89,"scoreLo":157.63,"scoreHi":164.82,"modelScoreStateId":"claude-opus-4-8::effort:max","modelScoreConfigSlug":"claude-opus-4-8-max","modelScoreStateName":"Opus 4.8 (max)","modelScoreStateBestProbability":0.9816666666666667,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":39.3519,"rankLo":33.0,"rankHi":46.0},{"slug":"glm-5-3","modelId":"glm-5-3","modelSlug":"glm-5-3","name":"GLM-5.3","organizationSlug":"zhipu-ai","n":60,"rankEligible":true,"score":160.85,"scoreLo":157.48,"scoreHi":164.83,"modelScoreStateId":"glm-5-3::default","modelScoreConfigSlug":"glm-5-3","modelScoreStateName":"GLM-5.3","modelScoreStateBestProbability":0.7038888888888889,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":39.8369,"rankLo":31.0,"rankHi":48.0},{"slug":"ling-3-1-flash","modelId":"ling-3-1-flash","modelSlug":"ling-3-1-flash","name":"Ling 3.1 Flash","organizationSlug":"inclusion-ai","n":8,"rankEligible":true,"score":160.85,"scoreLo":156.86,"scoreHi":165.42,"modelScoreStateId":"ling-3-1-flash::enabled","modelScoreConfigSlug":"ling-3-1-flash-reasoning","modelScoreStateName":"Ling 3.1 Flash (enabled)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":39.9522,"rankLo":22.0,"rankHi":63.0},{"slug":"atria-dawn-preview","modelId":"atria-dawn-preview","modelSlug":"atria-dawn-preview","name":"Atria Dawn Preview","organizationSlug":"shanghai-ai-lab","n":6,"rankEligible":true,"score":160.79,"scoreLo":156.49,"scoreHi":165.91,"modelScoreStateId":"atria-dawn-preview::default","modelScoreConfigSlug":"atria-dawn-preview-bfcl-v4","modelScoreStateName":"Atria Dawn Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":41.31,"rankLo":18.0,"rankHi":69.0},{"slug":"gpt-5-6-luna-pro","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro","organizationSlug":"openai","n":5,"rankEligible":true,"score":160.71,"scoreLo":156.71,"scoreHi":165.33,"modelScoreStateId":"gpt-5-6-luna-pro::default","modelScoreConfigSlug":"gpt-5-6-luna-pro","modelScoreStateName":"GPT-5.6 Luna Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":41.5917,"rankLo":22.0,"rankHi":66.0},{"slug":"ember-1","modelId":"ember-1","modelSlug":"ember-1","name":"Ember-1","organizationSlug":"fireworks","n":27,"rankEligible":true,"score":160.62,"scoreLo":157.26,"scoreHi":164.68,"modelScoreStateId":"ember-1::default","modelScoreConfigSlug":"ember-1","modelScoreStateName":"Ember-1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":42.2803,"rankLo":30.0,"rankHi":54.0},{"slug":"gemini-3-7-flash","modelId":"gemini-3-7-flash","modelSlug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","organizationSlug":"google","n":120,"rankEligible":true,"score":160.61,"scoreLo":157.36,"scoreHi":164.54,"modelScoreStateId":"gemini-3-7-flash::effort:high","modelScoreConfigSlug":"gemini-3-7-flash-high","modelScoreStateName":"Gemini 3.7 Flash (high)","modelScoreStateBestProbability":0.9194444444444444,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":42.6103,"rankLo":36.0,"rankHi":48.0},{"slug":"kimi-k3","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3","organizationSlug":"moonshot-ai","n":193,"rankEligible":true,"score":160.49,"scoreLo":157.24,"scoreHi":164.27,"modelScoreStateId":"kimi-k3::effort:max","modelScoreConfigSlug":"kimi-k3-max","modelScoreStateName":"Kimi K3 (max)","modelScoreStateBestProbability":0.49916666666666665,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":43.9697,"rankLo":38.0,"rankHi":49.0},{"slug":"hy4-preview","modelId":"hy4-preview","modelSlug":"hy4-preview","name":"Hy4 Preview","organizationSlug":"tencent","n":5,"rankEligible":true,"score":160.09,"scoreLo":156.24,"scoreHi":164.59,"modelScoreStateId":"hy4-preview::effort:high","modelScoreConfigSlug":"hy4-preview-high","modelScoreStateName":"Hy4 Preview (high)","modelScoreStateBestProbability":0.6986111111111111,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":47.9661,"rankLo":26.0,"rankHi":69.0},{"slug":"qwen3-8-flash-next","modelId":"qwen3-8-flash-next","modelSlug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":159.96,"scoreLo":156.39,"scoreHi":164.14,"modelScoreStateId":"qwen3-8-flash-next::enabled","modelScoreConfigSlug":"qwen3-8-flash-next-thinking","modelScoreStateName":"Qwen3.8-Flash-Next (thinking)","modelScoreStateBestProbability":0.6977777777777778,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":49.2528,"rankLo":32.0,"rankHi":66.0},{"slug":"step-5-preview","modelId":"step-5-preview","modelSlug":"step-5-preview","name":"Step 5 Preview","organizationSlug":"stepfun","n":14,"rankEligible":true,"score":159.93,"scoreLo":156.47,"scoreHi":164.0,"modelScoreStateId":"step-5-preview::effort:high","modelScoreConfigSlug":"step-5-preview-high","modelScoreStateName":"Step 5 Preview (high)","modelScoreStateBestProbability":0.7358333333333333,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":49.6678,"rankLo":33.0,"rankHi":65.0},{"slug":"mimo-v2-6-flash","modelId":"mimo-v2-6-flash","modelSlug":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash","organizationSlug":"xiaomi","n":41,"rankEligible":true,"score":159.92,"scoreLo":156.7,"scoreHi":163.81,"modelScoreStateId":"mimo-v2-6-flash::default","modelScoreConfigSlug":"mimo-v2-6-flash","modelScoreStateName":"MiMo-V2.6-Flash","modelScoreStateBestProbability":0.8186111111111111,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":49.7075,"rankLo":40.0,"rankHi":60.0},{"slug":"gpt-6-luna","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna","organizationSlug":"openai","n":111,"rankEligible":true,"score":159.9,"scoreLo":156.7,"scoreHi":163.68,"modelScoreStateId":"gpt-6-luna::effort:max","modelScoreConfigSlug":"gpt-6-luna-max","modelScoreStateName":"GPT-6 Luna (max)","modelScoreStateBestProbability":0.9969444444444444,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":49.9075,"rankLo":44.0,"rankHi":56.0},{"slug":"mistral-large-4-1t-a52b","modelId":"mistral-large-4-1t-a52b","modelSlug":"mistral-large-4-1t-a52b","name":"Mistral Large 4","organizationSlug":"mistral-ai","n":17,"rankEligible":true,"score":159.88,"scoreLo":156.49,"scoreHi":163.92,"modelScoreStateId":"mistral-large-4-1t-a52b::effort:high","modelScoreConfigSlug":"mistral-large-4-1t-a52b-high","modelScoreStateName":"Mistral Large 4 (high)","modelScoreStateBestProbability":0.7772222222222223,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":50.1956,"rankLo":37.0,"rankHi":64.0},{"slug":"qwen3-8-2-4t-a95b","modelId":"qwen3-8-2-4t-a95b","modelSlug":"qwen3-8-2-4t-a95b","name":"Qwen3.8 2.4T A95B","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":159.84,"scoreLo":156.03,"scoreHi":164.4,"modelScoreStateId":"qwen3-8-2-4t-a95b::effort:xhigh","modelScoreConfigSlug":"qwen3-8-2-4t-a95b-xhigh","modelScoreStateName":"Qwen3.8 2.4T A95B (xhigh)","modelScoreStateBestProbability":0.5761111111111111,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":50.5256,"rankLo":28.0,"rankHi":70.0},{"slug":"grok-4-5","modelId":"grok-4-5","modelSlug":"grok-4-5","name":"Grok 4.5","organizationSlug":"xai","n":36,"rankEligible":true,"score":159.66,"scoreLo":156.48,"scoreHi":163.42,"modelScoreStateId":"grok-4-5::default","modelScoreConfigSlug":"grok-4-5-unknown","modelScoreStateName":"Grok 4.5","modelScoreStateBestProbability":0.28888888888888886,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":52.4917,"rankLo":45.0,"rankHi":61.0},{"slug":"gpt-5-6-luna","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna","organizationSlug":"openai","n":120,"rankEligible":true,"score":159.46,"scoreLo":156.3,"scoreHi":163.16,"modelScoreStateId":"gpt-5-6-luna::effort:max","modelScoreConfigSlug":"gpt-5-6-luna-max","modelScoreStateName":"GPT-5.6 Luna (max)","modelScoreStateBestProbability":0.9816666666666667,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":54.615,"rankLo":48.0,"rankHi":61.0},{"slug":"deepseek-v4-pro-0813","modelId":"deepseek-v4-pro-0813","modelSlug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","organizationSlug":"deepseek","n":77,"rankEligible":true,"score":159.45,"scoreLo":156.3,"scoreHi":163.31,"modelScoreStateId":"deepseek-v4-pro-0813::effort:max","modelScoreConfigSlug":"deepseek-v4-pro-0813-max","modelScoreStateName":"DeepSeek V4 Pro 0813 (max)","modelScoreStateBestProbability":0.5741666666666667,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":54.7067,"rankLo":47.0,"rankHi":63.0},{"slug":"glm-5-3-flash","modelId":"glm-5-3-flash","modelSlug":"glm-5-3-flash","name":"GLM-5.3 Flash","organizationSlug":"zhipu-ai","n":42,"rankEligible":true,"score":159.24,"scoreLo":156.03,"scoreHi":163.0,"modelScoreStateId":"glm-5-3-flash::default","modelScoreConfigSlug":"glm-5-3-flash","modelScoreStateName":"GLM-5.3 Flash","modelScoreStateBestProbability":0.9586111111111111,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":57.12,"rankLo":48.0,"rankHi":66.0},{"slug":"muse-spark-1-2","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2","organizationSlug":"meta","n":11,"rankEligible":true,"score":159.13,"scoreLo":155.83,"scoreHi":163.1,"modelScoreStateId":"muse-spark-1-2::effort:high","modelScoreConfigSlug":"muse-spark-1-2-high","modelScoreStateName":"Muse Spark 1.2 (high)","modelScoreStateBestProbability":0.5127777777777778,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":57.9922,"rankLo":42.0,"rankHi":71.0},{"slug":"mistral-large-4","modelId":"mistral-large-4","modelSlug":"mistral-large-4","name":"Mistral Large 4 Preview","organizationSlug":"mistral-ai","n":14,"rankEligible":true,"score":159.1,"scoreLo":155.67,"scoreHi":162.95,"modelScoreStateId":"mistral-large-4::enabled","modelScoreConfigSlug":"mistral-large-4-thinking","modelScoreStateName":"Mistral Large 4 Preview (thinking)","modelScoreStateBestProbability":0.5241666666666667,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":58.4453,"rankLo":45.0,"rankHi":70.0},{"slug":"muse-spark-1-1","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1","organizationSlug":"meta","n":72,"rankEligible":true,"score":159.1,"scoreLo":156.0,"scoreHi":162.88,"modelScoreStateId":"muse-spark-1-1::effort:xhigh","modelScoreConfigSlug":"muse-spark-1-1-xhigh","modelScoreStateName":"Muse Spark 1.1 (xhigh)","modelScoreStateBestProbability":0.8677777777777778,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":58.6331,"rankLo":50.0,"rankHi":66.0},{"slug":"claude-sonnet-5","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5","organizationSlug":"anthropic","n":107,"rankEligible":true,"score":158.98,"scoreLo":155.85,"scoreHi":162.75,"modelScoreStateId":"claude-sonnet-5::effort:max","modelScoreConfigSlug":"claude-sonnet-5-max","modelScoreStateName":"Sonnet 5 (max)","modelScoreStateBestProbability":0.5491666666666667,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":59.9681,"rankLo":53.0,"rankHi":67.0},{"slug":"gpt-5-4","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4","organizationSlug":"openai","n":149,"rankEligible":true,"score":158.77,"scoreLo":155.73,"scoreHi":162.39,"modelScoreStateId":"gpt-5-4::effort:xhigh","modelScoreConfigSlug":"gpt-5-4-xhigh","modelScoreStateName":"GPT-5.4 (xhigh)","modelScoreStateBestProbability":0.5925,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":62.2764,"rankLo":56.0,"rankHi":68.0},{"slug":"nanbeige4-3-33b","modelId":"nanbeige4-3-33b","modelSlug":"nanbeige4-3-33b","name":"Nanbeige4.3-33B","organizationSlug":"nanbeige","n":17,"rankEligible":true,"score":158.7,"scoreLo":155.42,"scoreHi":162.59,"modelScoreStateId":"nanbeige4-3-33b::enabled","modelScoreConfigSlug":"nanbeige4-3-33b-thinking","modelScoreStateName":"Nanbeige4.3-33B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":62.3561,"rankLo":49.0,"rankHi":73.0},{"slug":"claude-opus-4-7","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7","organizationSlug":"anthropic","n":104,"rankEligible":true,"score":158.69,"scoreLo":155.6,"scoreHi":162.34,"modelScoreStateId":"claude-opus-4-7::effort:max","modelScoreConfigSlug":"claude-opus-4-7-max","modelScoreStateName":"Opus 4.7 (max)","modelScoreStateBestProbability":0.8727777777777778,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":63.0631,"rankLo":56.0,"rankHi":69.0},{"slug":"deepseek-v4-flash-vision-exp","modelId":"deepseek-v4-flash-vision-exp","modelSlug":"deepseek-v4-flash-vision-exp","name":"DeepSeek V4 Flash Vision Exp","organizationSlug":"deepseek","n":21,"rankEligible":true,"score":158.52,"scoreLo":155.28,"scoreHi":162.4,"modelScoreStateId":"deepseek-v4-flash-vision-exp::effort:max","modelScoreConfigSlug":"deepseek-v4-flash-vision-exp-max","modelScoreStateName":"DeepSeek V4 Flash Vision Exp (max)","modelScoreStateBestProbability":0.6291666666666667,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":63.9603,"rankLo":51.0,"rankHi":73.0},{"slug":"muse-spark-open","modelId":"muse-spark-open","modelSlug":"muse-spark-open","name":"Muse Spark Open","organizationSlug":"meta","n":49,"rankEligible":true,"score":158.45,"scoreLo":155.34,"scoreHi":162.08,"modelScoreStateId":"muse-spark-open::effort:xhigh","modelScoreConfigSlug":"muse-spark-open-xhigh","modelScoreStateName":"Muse Spark Open (xhigh)","modelScoreStateBestProbability":0.6061111111111112,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":65.0081,"rankLo":56.0,"rankHi":71.0},{"slug":"gemini-3-5-flash","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","organizationSlug":"google","n":23,"rankEligible":true,"score":158.42,"scoreLo":155.27,"scoreHi":162.2,"modelScoreStateId":"gemini-3-5-flash::enabled","modelScoreConfigSlug":"gemini-3-5-flash-pi","modelScoreStateName":"Gemini 3.5 Flash (enabled)","modelScoreStateBestProbability":0.9575,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":64.8731,"rankLo":52.0,"rankHi":73.0},{"slug":"qwen3-8-omni-flash","modelId":"qwen3-8-omni-flash","modelSlug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","organizationSlug":"alibaba","n":15,"rankEligible":true,"score":157.96,"scoreLo":154.29,"scoreHi":162.24,"modelScoreStateId":"qwen3-8-omni-flash::default","modelScoreConfigSlug":"qwen3-8-omni-flash","modelScoreStateName":"Qwen3.8-Omni-Flash","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":67.5886,"rankLo":47.0,"rankHi":80.0},{"slug":"agnes-2-5-pro-beta","modelId":"agnes-2-5-pro-beta","modelSlug":"agnes-2-5-pro-beta","name":"Agnes 2.5 Pro Beta","organizationSlug":"sapiens-ai","n":7,"rankEligible":true,"score":157.89,"scoreLo":154.03,"scoreHi":162.46,"modelScoreStateId":"agnes-2-5-pro-beta::default","modelScoreConfigSlug":"agnes-2-5-pro-beta","modelScoreStateName":"Agnes 2.5 Pro Beta","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":67.4411,"rankLo":44.0,"rankHi":84.0},{"slug":"deepseek-v4-flash-0731","modelId":"deepseek-v4-flash-0731","modelSlug":"deepseek-v4-flash-0731","name":"DeepSeek-V4-Flash-0731","organizationSlug":"deepseek","n":23,"rankEligible":true,"score":157.79,"scoreLo":154.73,"scoreHi":161.48,"modelScoreStateId":"deepseek-v4-flash-0731::effort:max","modelScoreConfigSlug":"deepseek-v4-flash-0731-max","modelScoreStateName":"DeepSeek-V4-Flash-0731 (max)","modelScoreStateBestProbability":0.7505555555555555,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":69.7669,"rankLo":62.0,"rankHi":76.0},{"slug":"gpt-5-2-pro","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro","organizationSlug":"openai","n":14,"rankEligible":true,"score":157.7,"scoreLo":154.51,"scoreHi":161.46,"modelScoreStateId":"gpt-5-2-pro::effort:xhigh","modelScoreConfigSlug":"gpt-5-2-pro-xhigh","modelScoreStateName":"GPT-5.2 Pro (xhigh)","modelScoreStateBestProbability":0.9983333333333333,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":70.0506,"rankLo":58.0,"rankHi":78.0},{"slug":"gemini-3-6-flash","modelId":"gemini-3-6-flash","modelSlug":"gemini-3-6-flash","name":"Gemini 3.6 Flash","organizationSlug":"google","n":38,"rankEligible":true,"score":157.38,"scoreLo":154.42,"scoreHi":161.05,"modelScoreStateId":"gemini-3-6-flash::effort:medium","modelScoreConfigSlug":"gemini-3-6-flash","modelScoreStateName":"Gemini 3.6 Flash (medium)","modelScoreStateBestProbability":0.7283333333333334,"modelScoreReasoningCondition":"effort:medium","modelScoreReasoningEffort":"medium","modelScoreHarness":"none","rankMean":72.2828,"rankLo":66.0,"rankHi":78.0},{"slug":"claude-opus-4-6","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6","organizationSlug":"anthropic","n":35,"rankEligible":true,"score":157.33,"scoreLo":154.36,"scoreHi":160.84,"modelScoreStateId":"claude-opus-4-6::effort:high","modelScoreConfigSlug":"claude-opus-4-6-high","modelScoreStateName":"Opus 4.6 (high)","modelScoreStateBestProbability":0.5525,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":72.5303,"rankLo":67.0,"rankHi":77.0},{"slug":"gpt-5-3-codex","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex","organizationSlug":"openai","n":26,"rankEligible":true,"score":157.31,"scoreLo":154.19,"scoreHi":161.07,"modelScoreStateId":"gpt-5-3-codex::effort:xhigh","modelScoreConfigSlug":"gpt-5-3-codex-xhigh","modelScoreStateName":"GPT-5.3-Codex (xhigh)","modelScoreStateBestProbability":0.8436111111111111,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":72.5197,"rankLo":64.0,"rankHi":79.0},{"slug":"mercury-3","modelId":"mercury-3","modelSlug":"mercury-3","name":"Mercury 3","organizationSlug":"inception-labs","n":14,"rankEligible":true,"score":157.15,"scoreLo":153.9,"scoreHi":160.98,"modelScoreStateId":"mercury-3::effort:high","modelScoreConfigSlug":"mercury-3-high","modelScoreStateName":"Mercury 3 (high)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":73.2189,"rankLo":62.0,"rankHi":82.0},{"slug":"glm-5-2","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2","organizationSlug":"zhipu-ai","n":55,"rankEligible":true,"score":156.9,"scoreLo":153.97,"scoreHi":160.37,"modelScoreStateId":"glm-5-2::default","modelScoreConfigSlug":"glm-5-2","modelScoreStateName":"GLM-5.2","modelScoreStateBestProbability":0.5836111111111111,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":74.9592,"rankLo":71.0,"rankHi":79.0},{"slug":"gemini-3-1-pro-preview","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview","organizationSlug":"google","n":121,"rankEligible":true,"score":156.36,"scoreLo":153.6,"scoreHi":159.75,"modelScoreStateId":"gemini-3-1-pro-preview::enabled","modelScoreConfigSlug":"gemini-3-1-pro","modelScoreStateName":"Gemini 3.1 Pro Preview (enabled)","modelScoreStateBestProbability":0.6216666666666667,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":77.5497,"rankLo":74.0,"rankHi":81.0},{"slug":"doubao-seed-2-1-pro-260628","modelId":"doubao-seed-2-1-pro-260628","modelSlug":"doubao-seed-2-1-pro-260628","name":"Doubao Seed 2.1 Pro","organizationSlug":"bytedance","n":5,"rankEligible":true,"score":155.84,"scoreLo":151.81,"scoreHi":160.37,"modelScoreStateId":"doubao-seed-2-1-pro-260628::default","modelScoreConfigSlug":"doubao-seed-2-1-pro-260628","modelScoreStateName":"Doubao Seed 2.1 Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":80.8808,"rankLo":63.0,"rankHi":104.0},{"slug":"qwen3-8-27b","modelId":"qwen3-8-27b","modelSlug":"qwen3-8-27b","name":"Qwen3.8 27B","organizationSlug":"alibaba","n":47,"rankEligible":true,"score":155.82,"scoreLo":152.99,"scoreHi":159.2,"modelScoreStateId":"qwen3-8-27b::default","modelScoreConfigSlug":"qwen3-8-27b","modelScoreStateName":"Qwen3.8 27B","modelScoreStateBestProbability":0.5769444444444445,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":80.0875,"rankLo":76.0,"rankHi":86.0},{"slug":"claude-sonnet-4-6","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6","organizationSlug":"anthropic","n":88,"rankEligible":true,"score":155.26,"scoreLo":152.51,"scoreHi":158.5,"modelScoreStateId":"claude-sonnet-4-6::effort:max","modelScoreConfigSlug":"claude-sonnet-4-6-max","modelScoreStateName":"Sonnet 4.6 (max)","modelScoreStateBestProbability":0.6983333333333334,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":83.0925,"rankLo":79.0,"rankHi":89.0},{"slug":"composer-2-5","modelId":"composer-2-5","modelSlug":"composer-2-5","name":"Composer 2.5","organizationSlug":"cursor","n":12,"rankEligible":true,"score":155.24,"scoreLo":152.31,"scoreHi":158.68,"modelScoreStateId":"composer-2-5::default","modelScoreConfigSlug":"composer-2-5","modelScoreStateName":"Composer 2.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":83.5406,"rankLo":77.0,"rankHi":94.0},{"slug":"gpt-5-2","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2","organizationSlug":"openai","n":87,"rankEligible":true,"score":155.23,"scoreLo":152.5,"scoreHi":158.47,"modelScoreStateId":"gpt-5-2::effort:xhigh","modelScoreConfigSlug":"gpt-5-2-xhigh","modelScoreStateName":"GPT-5.2 (xhigh)","modelScoreStateBestProbability":0.8883333333333333,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":83.2731,"rankLo":79.0,"rankHi":89.0},{"slug":"beam-1-0-501b-a23b","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","n":23,"rankEligible":true,"score":155.21,"scoreLo":152.2,"scoreHi":158.77,"modelScoreStateId":"beam-1-0-501b-a23b::effort:max","modelScoreConfigSlug":"beam-1-0-501b-a23b-max","modelScoreStateName":"Beam 1.0 501B A23B (max)","modelScoreStateBestProbability":0.7119444444444445,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":83.8028,"rankLo":76.0,"rankHi":95.0},{"slug":"qwen3-8-flash","modelId":"qwen3-8-flash","modelSlug":"qwen3-8-flash","name":"Qwen3.8 Flash","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":154.95,"scoreLo":150.15,"scoreHi":159.95,"modelScoreStateId":"qwen3-8-flash::default","modelScoreConfigSlug":"qwen3-8-flash","modelScoreStateName":"Qwen3.8 Flash","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":87.5017,"rankLo":61.975,"rankHi":120.0},{"slug":"muse-spark","modelId":"muse-spark","modelSlug":"muse-spark","name":"Muse Spark","organizationSlug":"meta","n":31,"rankEligible":true,"score":154.69,"scoreLo":151.9,"scoreHi":157.9,"modelScoreStateId":"muse-spark::enabled","modelScoreConfigSlug":"muse-spark-thinking","modelScoreStateName":"Muse Spark (thinking)","modelScoreStateBestProbability":0.97,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":87.1844,"rankLo":80.0,"rankHi":97.0},{"slug":"motif-3-beta","modelId":"motif-3-beta","modelSlug":"motif-3-beta","name":"Motif-3-Beta","organizationSlug":"motif","n":5,"rankEligible":true,"score":154.54,"scoreLo":150.61,"scoreHi":158.78,"modelScoreStateId":"motif-3-beta::default","modelScoreConfigSlug":"motif-3-beta","modelScoreStateName":"Motif-3-Beta","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":89.6811,"rankLo":72.975,"rankHi":115.0},{"slug":"gemini-2-5-deep-think","modelId":"gemini-2-5-deep-think","modelSlug":"gemini-2-5-deep-think","name":"Gemini 2.5 Deep Think","organizationSlug":"google","n":4,"rankEligible":true,"score":154.37,"scoreLo":150.28,"scoreHi":159.01,"modelScoreStateId":"gemini-2-5-deep-think::default","modelScoreConfigSlug":"gemini-2-5-deep-think","modelScoreStateName":"Gemini 2.5 Deep Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":91.3286,"rankLo":70.975,"rankHi":120.0},{"slug":"kimi-k2-7-code","modelId":"kimi-k2-7-code","modelSlug":"kimi-k2-7-code","name":"Kimi K2.7 Code","organizationSlug":"moonshot-ai","n":85,"rankEligible":true,"score":154.34,"scoreLo":151.73,"scoreHi":157.52,"modelScoreStateId":"kimi-k2-7-code::default","modelScoreConfigSlug":"kimi-k2-7-code","modelScoreStateName":"Kimi K2.7 Code","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":89.7425,"rankLo":84.0,"rankHi":98.0},{"slug":"apodex-1-1","modelId":"apodex-1-1","modelSlug":"apodex-1-1","name":"Apodex 1.1","organizationSlug":"apodex","n":11,"rankEligible":true,"score":154.18,"scoreLo":150.9,"scoreHi":157.93,"modelScoreStateId":"apodex-1-1::default","modelScoreConfigSlug":"apodex-1-1","modelScoreStateName":"Apodex 1.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":91.8547,"rankLo":78.0,"rankHi":111.0},{"slug":"gemini-3-pro-preview","modelId":"gemini-3-pro-preview","modelSlug":"gemini-3-pro-preview","name":"Gemini 3 Pro Preview","organizationSlug":"google","n":81,"rankEligible":true,"score":154.17,"scoreLo":151.59,"scoreHi":157.25,"modelScoreStateId":"gemini-3-pro-preview::effort:high","modelScoreConfigSlug":"gemini-3-pro-high","modelScoreStateName":"Gemini 3 Pro Preview (high)","modelScoreStateBestProbability":0.7363888888888889,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":91.2183,"rankLo":85.0,"rankHi":99.0},{"slug":"deepseek-v4-pro","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro","organizationSlug":"deepseek","n":89,"rankEligible":true,"score":154.08,"scoreLo":151.43,"scoreHi":157.1,"modelScoreStateId":"deepseek-v4-pro::effort:max","modelScoreConfigSlug":"deepseek-v4-pro-max","modelScoreStateName":"DeepSeek-V4-Pro (max)","modelScoreStateBestProbability":0.7130555555555556,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":92.0628,"rankLo":85.0,"rankHi":100.0},{"slug":"kimi-k2-6","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6","organizationSlug":"moonshot-ai","n":74,"rankEligible":true,"score":153.88,"scoreLo":151.33,"scoreHi":156.96,"modelScoreStateId":"kimi-k2-6::enabled","modelScoreConfigSlug":"kimi-k2-6-thinking","modelScoreStateName":"Kimi K2.6 (thinking)","modelScoreStateBestProbability":0.5527777777777778,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":93.8803,"rankLo":87.0,"rankHi":101.0},{"slug":"k2-horizon-375b-a23b","modelId":"k2-horizon-375b-a23b","modelSlug":"k2-horizon-375b-a23b","name":"K2 Horizon 375B A23B","organizationSlug":"llm360","n":13,"rankEligible":true,"score":153.75,"scoreLo":150.64,"scoreHi":157.39,"modelScoreStateId":"k2-horizon-375b-a23b::enabled","modelScoreConfigSlug":"k2-horizon-375b-a23b-thinking","modelScoreStateName":"K2 Horizon 375B A23B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":95.3847,"rankLo":80.0,"rankHi":113.0},{"slug":"hy3","modelId":"hy3","modelSlug":"hy3","name":"Hy3","organizationSlug":"tencent","n":12,"rankEligible":true,"score":153.7,"scoreLo":150.52,"scoreHi":157.34,"modelScoreStateId":"hy3::effort:high","modelScoreConfigSlug":"hy3-high","modelScoreStateName":"Hy3 (high)","modelScoreStateBestProbability":0.7591666666666667,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":95.8572,"rankLo":80.0,"rankHi":113.0},{"slug":"nemotron-3-5-super-vl-120b-a12b-bf16","modelId":"nemotron-3-5-super-vl-120b-a12b-bf16","modelSlug":"nemotron-3-5-super-vl-120b-a12b-bf16","name":"Nemotron 3.5 Super VL 120B A12B BF16","organizationSlug":"nvidia","n":18,"rankEligible":true,"score":153.54,"scoreLo":150.32,"scoreHi":157.32,"modelScoreStateId":"nemotron-3-5-super-vl-120b-a12b-bf16::default","modelScoreConfigSlug":"nemotron-3-5-super-vl-120b-a12b-bf16","modelScoreStateName":"Nemotron 3.5 Super VL 120B A12B BF16","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":97.2319,"rankLo":81.0,"rankHi":115.0},{"slug":"nex-n2-pro","modelId":"nex-n2-pro","modelSlug":"nex-n2-pro","name":"Nex-N2-Pro","organizationSlug":"nex-agi","n":14,"rankEligible":true,"score":153.52,"scoreLo":150.3,"scoreHi":157.27,"modelScoreStateId":"nex-n2-pro::default","modelScoreConfigSlug":"nex-n2-pro","modelScoreStateName":"Nex-N2-Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":97.4139,"rankLo":80.0,"rankHi":118.0},{"slug":"quasar-438b","modelId":"quasar-438b","modelSlug":"quasar-438b","name":"Quasar 438B","organizationSlug":"multiverse-computing","n":10,"rankEligible":true,"score":153.5,"scoreLo":150.19,"scoreHi":157.35,"modelScoreStateId":"quasar-438b::effort:max","modelScoreConfigSlug":"quasar-438b-max","modelScoreStateName":"Quasar 438B (max)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":97.6397,"rankLo":80.0,"rankHi":118.0},{"slug":"deepseek-v4-flash","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash","organizationSlug":"deepseek","n":37,"rankEligible":true,"score":153.41,"scoreLo":150.73,"scoreHi":156.48,"modelScoreStateId":"deepseek-v4-flash::effort:max","modelScoreConfigSlug":"deepseek-v4-flash-max","modelScoreStateName":"DeepSeek-V4-Flash (max)","modelScoreStateBestProbability":0.9838888888888889,"modelScoreReasoningCondition":"effort:max","modelScoreReasoningEffort":"max","modelScoreHarness":"none","rankMean":98.2308,"rankLo":88.0,"rankHi":108.0},{"slug":"nemotron-3-5-super-vl-120b-a12b-nvfp4","modelId":"nemotron-3-5-super-vl-120b-a12b-nvfp4","modelSlug":"nemotron-3-5-super-vl-120b-a12b-nvfp4","name":"Nemotron 3.5 Super VL 120B A12B NVFP4","organizationSlug":"nvidia","n":18,"rankEligible":true,"score":153.39,"scoreLo":150.27,"scoreHi":157.04,"modelScoreStateId":"nemotron-3-5-super-vl-120b-a12b-nvfp4::default","modelScoreConfigSlug":"nemotron-3-5-super-vl-120b-a12b-nvfp4","modelScoreStateName":"Nemotron 3.5 Super VL 120B A12B NVFP4","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":98.5681,"rankLo":82.0,"rankHi":116.0},{"slug":"motif-3","modelId":"motif-3","modelSlug":"motif-3","name":"Motif 3","organizationSlug":"motif","n":7,"rankEligible":true,"score":153.24,"scoreLo":149.66,"scoreHi":157.23,"modelScoreStateId":"motif-3::default","modelScoreConfigSlug":"motif-3","modelScoreStateName":"Motif 3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":100.0689,"rankLo":80.0,"rankHi":127.0},{"slug":"minimax-m3","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3","organizationSlug":"minimax","n":29,"rankEligible":true,"score":153.2,"scoreLo":150.46,"scoreHi":156.23,"modelScoreStateId":"minimax-m3::enabled","modelScoreConfigSlug":"minimax-m3-thinking","modelScoreStateName":"MiniMax M3 (thinking)","modelScoreStateBestProbability":0.6458333333333334,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":100.1469,"rankLo":88.975,"rankHi":111.0},{"slug":"qwen3-6-max-preview","modelId":"qwen3-6-max-preview","modelSlug":"qwen3-6-max-preview","name":"Qwen3.6-Max-Preview","organizationSlug":"alibaba","n":26,"rankEligible":true,"score":153.17,"scoreLo":150.37,"scoreHi":156.49,"modelScoreStateId":"qwen3-6-max-preview::default","modelScoreConfigSlug":"qwen3-6-max","modelScoreStateName":"Qwen3.6-Max-Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":100.3994,"rankLo":87.0,"rankHi":114.0},{"slug":"claude-opus-4-5-20251101","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5","organizationSlug":"anthropic","n":30,"rankEligible":true,"score":153.02,"scoreLo":150.31,"scoreHi":156.19,"modelScoreStateId":"claude-opus-4-5-20251101::effort:high","modelScoreConfigSlug":"claude-opus-4-5-20251101-high","modelScoreStateName":"Opus 4.5 (high)","modelScoreStateBestProbability":0.7894444444444444,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":101.7483,"rankLo":90.0,"rankHi":113.0},{"slug":"gpt-5-pro","modelId":"gpt-5-pro","modelSlug":"gpt-5-pro","name":"GPT-5 Pro","organizationSlug":"openai","n":18,"rankEligible":true,"score":152.98,"scoreLo":150.07,"scoreHi":156.28,"modelScoreStateId":"gpt-5-pro::effort:high","modelScoreConfigSlug":"gpt-5-pro-high","modelScoreStateName":"GPT-5 Pro (high)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":102.1481,"rankLo":87.0,"rankHi":118.0},{"slug":"laguna-s-2-1","modelId":"laguna-s-2-1","modelSlug":"laguna-s-2-1","name":"Laguna S 2.1","organizationSlug":"poolside","n":8,"rankEligible":true,"score":152.88,"scoreLo":149.08,"scoreHi":156.99,"modelScoreStateId":"laguna-s-2-1::enabled","modelScoreConfigSlug":"laguna-s-2-1-thinking","modelScoreStateName":"Laguna S 2.1 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":103.4567,"rankLo":80.0,"rankHi":133.0},{"slug":"glm-5-1","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1","organizationSlug":"zhipu-ai","n":77,"rankEligible":true,"score":152.84,"scoreLo":150.36,"scoreHi":155.75,"modelScoreStateId":"glm-5-1::default","modelScoreConfigSlug":"glm-5-1","modelScoreStateName":"GLM-5.1","modelScoreStateBestProbability":0.4286111111111111,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":103.4172,"rankLo":96.0,"rankHi":111.0},{"slug":"gemini-3-flash-preview","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview","organizationSlug":"google","n":77,"rankEligible":true,"score":152.57,"scoreLo":150.06,"scoreHi":155.44,"modelScoreStateId":"gemini-3-flash-preview::enabled","modelScoreConfigSlug":"gemini-3-flash","modelScoreStateName":"Gemini 3 Flash Preview (enabled)","modelScoreStateBestProbability":0.635,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":105.8397,"rankLo":98.0,"rankHi":114.0},{"slug":"gpt-5-2-codex","modelId":"gpt-5-2-codex","modelSlug":"gpt-5-2-codex","name":"GPT-5.2-Codex","organizationSlug":"openai","n":8,"rankEligible":true,"score":152.44,"scoreLo":149.4,"scoreHi":155.82,"modelScoreStateId":"gpt-5-2-codex::effort:high","modelScoreConfigSlug":"gpt-5-2-codex-high","modelScoreStateName":"GPT-5.2-Codex (high)","modelScoreStateBestProbability":0.45666666666666667,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":107.0306,"rankLo":89.0,"rankHi":128.0},{"slug":"gpt-5-1-codex-max","modelId":"gpt-5-1-codex-max","modelSlug":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max","organizationSlug":"openai","n":6,"rankEligible":true,"score":152.22,"scoreLo":148.8,"scoreHi":156.17,"modelScoreStateId":"gpt-5-1-codex-max::effort:xhigh","modelScoreConfigSlug":"gpt-5-1-codex-max","modelScoreStateName":"GPT-5.1-Codex-Max (xhigh)","modelScoreStateBestProbability":0.7822222222222223,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":109.3308,"rankLo":85.0,"rankHi":136.0},{"slug":"inkling-small","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small","organizationSlug":"thinking-machines-lab","n":59,"rankEligible":true,"score":151.84,"scoreLo":149.42,"scoreHi":154.58,"modelScoreStateId":"inkling-small::effort:0.99","modelScoreConfigSlug":"inkling-small-effort-0-99","modelScoreStateName":"Inkling-Small (0.99)","modelScoreStateBestProbability":0.7027777777777777,"modelScoreReasoningCondition":"effort:0.99","modelScoreReasoningEffort":"0.99","modelScoreHarness":"none","rankMean":112.1944,"rankLo":104.0,"rankHi":121.0},{"slug":"qwen3-6-plus","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":151.69,"scoreLo":148.93,"scoreHi":154.98,"modelScoreStateId":"qwen3-6-plus::enabled","modelScoreConfigSlug":"qwen3-6-plus-thinking","modelScoreStateName":"Qwen3.6 Plus (2026-04-02) (thinking)","modelScoreStateBestProbability":0.6202777777777778,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":114.0036,"rankLo":98.0,"rankHi":133.025},{"slug":"inkling-small-preview","modelId":"inkling-small-preview","modelSlug":"inkling-small-preview","name":"Inkling-Small Preview","organizationSlug":"thinking-machines-lab","n":12,"rankEligible":true,"score":151.68,"scoreLo":148.35,"scoreHi":155.4,"modelScoreStateId":"inkling-small-preview::effort:0.99","modelScoreConfigSlug":"inkling-small-preview-effort-0-99","modelScoreStateName":"Inkling-Small Preview (0.99)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:0.99","modelScoreReasoningEffort":"0.99","modelScoreHarness":"none","rankMean":114.3714,"rankLo":92.0,"rankHi":139.0},{"slug":"gpt-5-4-mini","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini","organizationSlug":"openai","n":74,"rankEligible":true,"score":151.6,"scoreLo":149.15,"scoreHi":154.41,"modelScoreStateId":"gpt-5-4-mini::effort:xhigh","modelScoreConfigSlug":"gpt-5-4-mini-xhigh","modelScoreStateName":"GPT-5.4 Mini (xhigh)","modelScoreStateBestProbability":0.9472222222222222,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":114.4147,"rankLo":106.0,"rankHi":124.0},{"slug":"ling-3-0-flash-vl","modelId":"ling-3-0-flash-vl","modelSlug":"ling-3-0-flash-vl","name":"Ling 3.0 Flash VL","organizationSlug":"inclusion-ai","n":11,"rankEligible":true,"score":151.49,"scoreLo":148.34,"scoreHi":155.14,"modelScoreStateId":"ling-3-0-flash-vl::enabled","modelScoreConfigSlug":"ling-3-0-flash-vl-thinking","modelScoreStateName":"Ling 3.0 Flash VL (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":116.1964,"rankLo":95.0,"rankHi":140.0},{"slug":"inkling","modelId":"inkling","modelSlug":"inkling","name":"Inkling","organizationSlug":"thinking-machines-lab","n":67,"rankEligible":true,"score":151.37,"scoreLo":149.04,"scoreHi":154.14,"modelScoreStateId":"inkling::effort:0.99","modelScoreConfigSlug":"inkling-effort-0-99","modelScoreStateName":"Inkling (0.99)","modelScoreStateBestProbability":0.6663888888888889,"modelScoreReasoningCondition":"effort:0.99","modelScoreReasoningEffort":"0.99","modelScoreHarness":"none","rankMean":116.6108,"rankLo":108.0,"rankHi":126.0},{"slug":"gemini-3-5-flash-lite","modelId":"gemini-3-5-flash-lite","modelSlug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","organizationSlug":"google","n":16,"rankEligible":true,"score":151.22,"scoreLo":148.46,"scoreHi":154.46,"modelScoreStateId":"gemini-3-5-flash-lite::enabled","modelScoreConfigSlug":"gemini-3-5-flash-lite-artificial-analysis","modelScoreStateName":"Gemini 3.5 Flash-Lite (enabled)","modelScoreStateBestProbability":0.8825,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":118.6586,"rankLo":102.0,"rankHi":138.0},{"slug":"mimo-v2-5-pro","modelId":"mimo-v2-5-pro","modelSlug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro","organizationSlug":"xiaomi","n":37,"rankEligible":true,"score":151.12,"scoreLo":148.57,"scoreHi":154.04,"modelScoreStateId":"mimo-v2-5-pro::default","modelScoreConfigSlug":"mimo-v2-5-pro","modelScoreStateName":"MiMo-V2.5-Pro","modelScoreStateBestProbability":0.5077777777777778,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":119.3992,"rankLo":107.0,"rankHi":134.0},{"slug":"ternary-bonsai-2-27b-gguf","modelId":"ternary-bonsai-2-27b-gguf","modelSlug":"ternary-bonsai-2-27b-gguf","name":"Ternary Bonsai 2 27B (GGUF)","organizationSlug":"prism-ml","n":4,"rankEligible":true,"score":151.06,"scoreLo":144.39,"scoreHi":158.55,"modelScoreStateId":"ternary-bonsai-2-27b-gguf::enabled","modelScoreConfigSlug":"ternary-bonsai-2-27b-gguf-thinking-evalscope-vllm","modelScoreStateName":"Ternary Bonsai 2 27B (GGUF) (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":123.1408,"rankLo":68.0,"rankHi":189.0},{"slug":"doubao-seed-2-0-pro-260215","modelId":"doubao-seed-2-0-pro-260215","modelSlug":"doubao-seed-2-0-pro-260215","name":"Doubao Seed 2.0 Pro","organizationSlug":"bytedance","n":9,"rankEligible":true,"score":151.05,"scoreLo":148.03,"scoreHi":154.57,"modelScoreStateId":"doubao-seed-2-0-pro-260215::default","modelScoreConfigSlug":"doubao-seed-2-0-pro-260215","modelScoreStateName":"Doubao Seed 2.0 Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":120.7019,"rankLo":99.0,"rankHi":146.0},{"slug":"grok-build-0-1","modelId":"grok-build-0-1","modelSlug":"grok-build-0-1","name":"Grok Build 0.1","organizationSlug":"xai","n":14,"rankEligible":true,"score":150.84,"scoreLo":147.81,"scoreHi":154.02,"modelScoreStateId":"grok-build-0-1::default","modelScoreConfigSlug":"grok-build-0-1","modelScoreStateName":"Grok Build 0.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":122.7228,"rankLo":104.0,"rankHi":146.0},{"slug":"glm-5","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5","organizationSlug":"zhipu-ai","n":32,"rankEligible":true,"score":150.82,"scoreLo":148.48,"scoreHi":153.56,"modelScoreStateId":"glm-5::enabled","modelScoreConfigSlug":"glm-5-thinking","modelScoreStateName":"GLM-5 (thinking)","modelScoreStateBestProbability":0.9994444444444445,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":122.5764,"rankLo":113.0,"rankHi":133.0},{"slug":"kimi-k2-5","modelId":"kimi-k2-5","modelSlug":"kimi-k2-5","name":"Kimi K2.5","organizationSlug":"moonshot-ai","n":90,"rankEligible":true,"score":150.74,"scoreLo":148.46,"scoreHi":153.38,"modelScoreStateId":"kimi-k2-5::enabled","modelScoreConfigSlug":"kimi-k2-5-thinking","modelScoreStateName":"Kimi K2.5 (thinking)","modelScoreStateBestProbability":0.8691666666666666,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":123.2983,"rankLo":115.0,"rankHi":133.0},{"slug":"qwen3-5-397b-a17b","modelId":"qwen3-5-397b-a17b","modelSlug":"qwen3-5-397b-a17b","name":"Qwen3.5 397B A17B","organizationSlug":"alibaba","n":111,"rankEligible":true,"score":150.57,"scoreLo":148.28,"scoreHi":153.28,"modelScoreStateId":"qwen3-5-397b-a17b::default","modelScoreConfigSlug":"qwen3-5-397b-a17b","modelScoreStateName":"Qwen3.5 397B A17B","modelScoreStateBestProbability":0.6213888888888889,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":125.3886,"rankLo":116.0,"rankHi":135.0},{"slug":"gpt-5-1","modelId":"gpt-5-1","modelSlug":"gpt-5-1","name":"GPT-5.1","organizationSlug":"openai","n":89,"rankEligible":true,"score":150.43,"scoreLo":148.15,"scoreHi":153.05,"modelScoreStateId":"gpt-5-1::effort:high","modelScoreConfigSlug":"gpt-5-1-high","modelScoreStateName":"GPT-5.1 (high)","modelScoreStateBestProbability":0.6927777777777778,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":126.9311,"rankLo":118.0,"rankHi":136.0},{"slug":"solar-mini4-260922","modelId":"solar-mini4-260922","modelSlug":"solar-mini4-260922","name":"Solar Mini 4","organizationSlug":"upstage","n":8,"rankEligible":true,"score":150.43,"scoreLo":146.96,"scoreHi":154.32,"modelScoreStateId":"solar-mini4-260922::enabled","modelScoreConfigSlug":"solar-mini4-260922-thinking","modelScoreStateName":"Solar Mini 4 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":127.2647,"rankLo":102.0,"rankHi":158.0},{"slug":"muse-glimmer-30b","modelId":"muse-glimmer-30b","modelSlug":"muse-glimmer-30b","name":"Muse Glimmer 30B","organizationSlug":"meta","n":25,"rankEligible":true,"score":150.39,"scoreLo":147.74,"scoreHi":153.42,"modelScoreStateId":"muse-glimmer-30b::effort:xhigh","modelScoreConfigSlug":"muse-glimmer-30b-xhigh","modelScoreStateName":"Muse Glimmer 30B (xhigh)","modelScoreStateBestProbability":0.6191666666666666,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":127.3972,"rankLo":112.0,"rankHi":144.0},{"slug":"k2-horizon-mova-36b-a4b","modelId":"k2-horizon-mova-36b-a4b","modelSlug":"k2-horizon-mova-36b-a4b","name":"K2 Horizon MoVA 36B A4B","organizationSlug":"llm360","n":9,"rankEligible":true,"score":150.38,"scoreLo":146.94,"scoreHi":154.13,"modelScoreStateId":"k2-horizon-mova-36b-a4b::enabled","modelScoreConfigSlug":"k2-horizon-mova-36b-a4b-thinking","modelScoreStateName":"K2 Horizon MoVA 36B A4B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":127.7536,"rankLo":103.0,"rankHi":157.025},{"slug":"solar-open2-250b","modelId":"solar-open2-250b","modelSlug":"solar-open2-250b","name":"Solar Open2 250B","organizationSlug":"upstage","n":5,"rankEligible":true,"score":150.36,"scoreLo":146.19,"scoreHi":154.91,"modelScoreStateId":"solar-open2-250b::enabled","modelScoreConfigSlug":"solar-open2-250b-thinking","modelScoreStateName":"Solar Open2 250B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":128.3114,"rankLo":95.0,"rankHi":168.0},{"slug":"grok-4-20","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20","organizationSlug":"xai","n":72,"rankEligible":true,"score":150.26,"scoreLo":148.01,"scoreHi":152.85,"modelScoreStateId":"grok-4-20::enabled","modelScoreConfigSlug":"grok-4-20-thinking","modelScoreStateName":"Grok 4.20 (thinking)","modelScoreStateBestProbability":0.9469444444444445,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":128.9317,"rankLo":118.0,"rankHi":140.0},{"slug":"ling-3-0-flash-fin","modelId":"ling-3-0-flash-fin","modelSlug":"ling-3-0-flash-fin","name":"Ling 3.0 Flash Fin","organizationSlug":"inclusion-ai","n":11,"rankEligible":true,"score":150.21,"scoreLo":146.97,"scoreHi":153.86,"modelScoreStateId":"ling-3-0-flash-fin::enabled","modelScoreConfigSlug":"ling-3-0-flash-fin-thinking","modelScoreStateName":"Ling 3.0 Flash Fin (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":129.5356,"rankLo":106.0,"rankHi":157.0},{"slug":"mimo-v2-5","modelId":"mimo-v2-5","modelSlug":"mimo-v2-5","name":"MiMo-V2.5","organizationSlug":"xiaomi","n":38,"rankEligible":true,"score":150.17,"scoreLo":147.72,"scoreHi":152.92,"modelScoreStateId":"mimo-v2-5::default","modelScoreConfigSlug":"mimo-v2-5","modelScoreStateName":"MiMo-V2.5","modelScoreStateBestProbability":0.5275,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":129.7853,"rankLo":116.0,"rankHi":144.0},{"slug":"hy3-preview","modelId":"hy3-preview","modelSlug":"hy3-preview","name":"Hy3 preview","organizationSlug":"tencent","n":4,"rankEligible":true,"score":150.14,"scoreLo":146.96,"scoreHi":153.67,"modelScoreStateId":"hy3-preview::effort:high","modelScoreConfigSlug":"hy3-preview-high","modelScoreStateName":"Hy3 preview (high)","modelScoreStateBestProbability":0.4766666666666667,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":130.2489,"rankLo":106.0,"rankHi":158.0},{"slug":"gpt-5","modelId":"gpt-5","modelSlug":"gpt-5","name":"GPT-5","organizationSlug":"openai","n":11,"rankEligible":true,"score":150.0,"scoreLo":150.0,"scoreHi":150.0,"modelScoreStateId":"gpt-5::enabled","modelScoreConfigSlug":"gpt-5-thinking","modelScoreStateName":"GPT-5 (thinking)","modelScoreStateBestProbability":0.4725,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":130.9883,"rankLo":108.0,"rankHi":155.025},{"slug":"gpt-5-codex","modelId":"gpt-5-codex","modelSlug":"gpt-5-codex","name":"GPT-5-Codex","organizationSlug":"openai","n":13,"rankEligible":true,"score":149.94,"scoreLo":146.95,"scoreHi":153.28,"modelScoreStateId":"gpt-5-codex::effort:high","modelScoreConfigSlug":"gpt-5-codex-high","modelScoreStateName":"GPT-5-Codex (high)","modelScoreStateBestProbability":0.9172222222222223,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":132.4039,"rankLo":110.0,"rankHi":156.0},{"slug":"kat-coder-pro-v2","modelId":"kat-coder-pro-v2","modelSlug":"kat-coder-pro-v2","name":"KAT-Coder-Pro V2","organizationSlug":"kuaishou","n":11,"rankEligible":true,"score":149.83,"scoreLo":146.41,"scoreHi":153.54,"modelScoreStateId":"kat-coder-pro-v2::default","modelScoreConfigSlug":"kat-coder-pro-v2","modelScoreStateName":"KAT-Coder-Pro V2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":133.6628,"rankLo":106.0,"rankHi":166.0},{"slug":"gpt-5-1-codex","modelId":"gpt-5-1-codex","modelSlug":"gpt-5-1-codex","name":"GPT-5.1-Codex","organizationSlug":"openai","n":16,"rankEligible":true,"score":149.64,"scoreLo":146.96,"scoreHi":152.66,"modelScoreStateId":"gpt-5-1-codex::effort:high","modelScoreConfigSlug":"gpt-5-1-codex-high","modelScoreStateName":"GPT-5.1-Codex (high)","modelScoreStateBestProbability":0.9794444444444445,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":135.7006,"rankLo":116.0,"rankHi":155.0},{"slug":"qwen3-6-27b","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B","organizationSlug":"alibaba","n":61,"rankEligible":true,"score":149.64,"scoreLo":147.31,"scoreHi":152.5,"modelScoreStateId":"qwen3-6-27b::default","modelScoreConfigSlug":"qwen3-6-27b","modelScoreStateName":"Qwen3.6 27B","modelScoreStateBestProbability":0.5594444444444444,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":135.7072,"rankLo":123.0,"rankHi":148.0},{"slug":"grok-4-3","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3","organizationSlug":"xai","n":9,"rankEligible":true,"score":149.55,"scoreLo":146.7,"scoreHi":152.85,"modelScoreStateId":"grok-4-3::effort:medium","modelScoreConfigSlug":"grok-4-3-medium","modelScoreStateName":"Grok 4.3 (medium)","modelScoreStateBestProbability":0.6775,"modelScoreReasoningCondition":"effort:medium","modelScoreReasoningEffort":"medium","modelScoreHarness":"none","rankMean":136.6103,"rankLo":114.0,"rankHi":160.025},{"slug":"deepseek-v3-2-speciale","modelId":"deepseek-v3-2-speciale","modelSlug":"deepseek-v3-2-speciale","name":"DeepSeek-V3.2-Speciale","organizationSlug":"deepseek","n":23,"rankEligible":true,"score":149.22,"scoreLo":146.31,"scoreHi":152.52,"modelScoreStateId":"deepseek-v3-2-speciale::default","modelScoreConfigSlug":"deepseek-v3-2-speciale-thinking","modelScoreStateName":"DeepSeek-V3.2-Speciale","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":140.2375,"rankLo":118.0,"rankHi":163.0},{"slug":"qwen3-5-plus","modelId":"qwen3-5-plus","modelSlug":"qwen3-5-plus","name":"Qwen3.5 Plus (2026-02-15)","organizationSlug":"alibaba","n":20,"rankEligible":true,"score":149.13,"scoreLo":146.49,"scoreHi":152.19,"modelScoreStateId":"qwen3-5-plus::enabled","modelScoreConfigSlug":"qwen3-5-plus-thinking","modelScoreStateName":"Qwen3.5 Plus (2026-02-15) (thinking)","modelScoreStateBestProbability":0.6536111111111111,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":141.1725,"rankLo":120.0,"rankHi":162.0},{"slug":"qwen3-5-122b-a10b","modelId":"qwen3-5-122b-a10b","modelSlug":"qwen3-5-122b-a10b","name":"Qwen3.5 122B A10B","organizationSlug":"alibaba","n":79,"rankEligible":true,"score":149.11,"scoreLo":146.86,"scoreHi":151.72,"modelScoreStateId":"qwen3-5-122b-a10b::default","modelScoreConfigSlug":"qwen3-5-122b-a10b","modelScoreStateName":"Qwen3.5 122B A10B","modelScoreStateBestProbability":0.6252777777777778,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":141.4,"rankLo":130.0,"rankHi":153.0},{"slug":"step-3-7-flash","modelId":"step-3-7-flash","modelSlug":"step-3-7-flash","name":"Step 3.7 Flash","organizationSlug":"stepfun","n":12,"rankEligible":true,"score":149.11,"scoreLo":146.56,"scoreHi":151.94,"modelScoreStateId":"step-3-7-flash::effort:high","modelScoreConfigSlug":"step-3-7-flash-high","modelScoreStateName":"Step 3.7 Flash (high)","modelScoreStateBestProbability":0.9577777777777777,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":141.3289,"rankLo":122.0,"rankHi":160.0},{"slug":"step-3-5-flash","modelId":"step-3-5-flash","modelSlug":"step-3-5-flash","name":"Step 3.5 Flash","organizationSlug":"stepfun","n":16,"rankEligible":true,"score":149.02,"scoreLo":146.08,"scoreHi":152.2,"modelScoreStateId":"step-3-5-flash::enabled","modelScoreConfigSlug":"step-3-5-flash-thinking","modelScoreStateName":"Step 3.5 Flash (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":142.3981,"rankLo":119.0,"rankHi":167.0},{"slug":"qwen3-5-27b","modelId":"qwen3-5-27b","modelSlug":"qwen3-5-27b","name":"Qwen3.5-27B","organizationSlug":"alibaba","n":87,"rankEligible":true,"score":149.01,"scoreLo":146.81,"scoreHi":151.57,"modelScoreStateId":"qwen3-5-27b::default","modelScoreConfigSlug":"qwen-3-5-27b","modelScoreStateName":"Qwen3.5-27B","modelScoreStateBestProbability":0.76,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":142.4844,"rankLo":131.0,"rankHi":154.025},{"slug":"o3-pro","modelId":"o3-pro","modelSlug":"o3-pro","name":"o3-pro","organizationSlug":"openai","n":9,"rankEligible":true,"score":148.9,"scoreLo":146.16,"scoreHi":151.85,"modelScoreStateId":"o3-pro::effort:high","modelScoreConfigSlug":"o3-pro-high","modelScoreStateName":"o3-pro (high)","modelScoreStateBestProbability":0.5675,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":143.6564,"rankLo":124.0,"rankHi":164.0},{"slug":"glm-5v-turbo","modelId":"glm-5v-turbo","modelSlug":"glm-5v-turbo","name":"GLM 5V Turbo","organizationSlug":"zhipu-ai","n":20,"rankEligible":true,"score":148.7,"scoreLo":146.02,"scoreHi":151.72,"modelScoreStateId":"glm-5v-turbo::default","modelScoreConfigSlug":"glm-5v-turbo","modelScoreStateName":"GLM 5V Turbo","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":145.9911,"rankLo":126.0,"rankHi":167.0},{"slug":"nemotron-3-ultra-550b-a55b-bf16","modelId":"nemotron-3-ultra-550b-a55b-bf16","modelSlug":"nemotron-3-ultra-550b-a55b-bf16","name":"Nemotron 3 Ultra 550B A55B","organizationSlug":"nvidia","n":26,"rankEligible":true,"score":148.64,"scoreLo":146.25,"scoreHi":151.34,"modelScoreStateId":"nemotron-3-ultra-550b-a55b-bf16::enabled","modelScoreConfigSlug":"nemotron-3-ultra-550b-a55b-bf16-thinking","modelScoreStateName":"Nemotron 3 Ultra 550B A55B (thinking)","modelScoreStateBestProbability":0.6041666666666666,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":146.4444,"rankLo":131.0,"rankHi":164.0},{"slug":"o3","modelId":"o3","modelSlug":"o3","name":"o3","organizationSlug":"openai","n":79,"rankEligible":true,"score":148.31,"scoreLo":146.21,"scoreHi":150.7,"modelScoreStateId":"o3::effort:high","modelScoreConfigSlug":"o3-high","modelScoreStateName":"o3 (high)","modelScoreStateBestProbability":0.8152777777777778,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":150.0744,"rankLo":140.975,"rankHi":161.0},{"slug":"gpt-5-4-nano","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano","organizationSlug":"openai","n":40,"rankEligible":true,"score":148.24,"scoreLo":145.99,"scoreHi":150.78,"modelScoreStateId":"gpt-5-4-nano::effort:xhigh","modelScoreConfigSlug":"gpt-5-4-nano-xhigh","modelScoreStateName":"GPT-5.4 Nano (xhigh)","modelScoreStateBestProbability":0.9752777777777778,"modelScoreReasoningCondition":"effort:xhigh","modelScoreReasoningEffort":"xhigh","modelScoreHarness":"none","rankMean":150.9689,"rankLo":139.0,"rankHi":164.0},{"slug":"step-3-5-flash-2603","modelId":"step-3-5-flash-2603","modelSlug":"step-3-5-flash-2603","name":"Step 3.5 Flash 2603","organizationSlug":"stepfun","n":5,"rankEligible":true,"score":148.23,"scoreLo":143.51,"scoreHi":153.02,"modelScoreStateId":"step-3-5-flash-2603::default","modelScoreConfigSlug":"step-3-5-flash-2603","modelScoreStateName":"Step 3.5 Flash 2603","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":151.1303,"rankLo":109.0,"rankHi":197.0},{"slug":"chat-latest-2026-05-05","modelId":"chat-latest-2026-05-05","modelSlug":"chat-latest-2026-05-05","name":"GPT-5.5 Instant","organizationSlug":"openai","n":14,"rankEligible":true,"score":148.2,"scoreLo":145.14,"scoreHi":151.46,"modelScoreStateId":"chat-latest-2026-05-05::default","modelScoreConfigSlug":"gpt-5-5-instant-05-26","modelScoreStateName":"GPT-5.5 Instant","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":151.535,"rankLo":127.0,"rankHi":177.0},{"slug":"glm-4-7","modelId":"glm-4-7","modelSlug":"glm-4-7","name":"GLM-4.7","organizationSlug":"zhipu-ai","n":12,"rankEligible":true,"score":148.0,"scoreLo":145.3,"scoreHi":150.92,"modelScoreStateId":"glm-4-7::enabled","modelScoreConfigSlug":"glm-4-7-thinking","modelScoreStateName":"GLM-4.7 (thinking)","modelScoreStateBestProbability":0.9691666666666666,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":153.6408,"rankLo":133.0,"rankHi":174.0},{"slug":"glm-5-turbo","modelId":"glm-5-turbo","modelSlug":"glm-5-turbo","name":"GLM 5 Turbo","organizationSlug":"zhipu-ai","n":6,"rankEligible":true,"score":147.98,"scoreLo":144.37,"scoreHi":151.72,"modelScoreStateId":"glm-5-turbo::enabled","modelScoreConfigSlug":"glm-5-turbo-thinking","modelScoreStateName":"GLM 5 Turbo (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":153.7683,"rankLo":121.0,"rankHi":188.0},{"slug":"gemma-4-31b-it","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT","organizationSlug":"google","n":11,"rankEligible":true,"score":147.86,"scoreLo":145.18,"scoreHi":150.77,"modelScoreStateId":"gemma-4-31b-it::effort:high","modelScoreConfigSlug":"gemma-4-31b-it-high","modelScoreStateName":"Gemma 4 31B IT (high)","modelScoreStateBestProbability":0.5091666666666667,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":155.2289,"rankLo":134.0,"rankHi":176.0},{"slug":"ling-3-0-flash","modelId":"ling-3-0-flash","modelSlug":"ling-3-0-flash","name":"Ling 3.0 Flash","organizationSlug":"inclusion-ai","n":11,"rankEligible":true,"score":147.79,"scoreLo":144.9,"scoreHi":150.86,"modelScoreStateId":"ling-3-0-flash::enabled","modelScoreConfigSlug":"ling-3-0-flash-thinking","modelScoreStateName":"Ling 3.0 Flash (thinking)","modelScoreStateBestProbability":0.9427777777777778,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":155.9606,"rankLo":132.0,"rankHi":179.025},{"slug":"qwen3-max-2026-01-23","modelId":"qwen3-max-2026-01-23","modelSlug":"qwen3-max-2026-01-23","name":"Qwen3 Max Thinking (2026-01-23)","organizationSlug":"alibaba","n":40,"rankEligible":true,"score":147.75,"scoreLo":145.52,"scoreHi":150.33,"modelScoreStateId":"qwen3-max-2026-01-23::enabled","modelScoreConfigSlug":"qwen3-max-thinking-2026-01-23","modelScoreStateName":"Qwen3 Max Thinking (2026-01-23) (thinking)","modelScoreStateBestProbability":0.9986111111111111,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":156.4492,"rankLo":142.0,"rankHi":170.025},{"slug":"deepseek-v3-2","modelId":"deepseek-v3-2","modelSlug":"deepseek-v3-2","name":"DeepSeek-V3.2","organizationSlug":"deepseek","n":52,"rankEligible":true,"score":147.74,"scoreLo":145.69,"scoreHi":150.09,"modelScoreStateId":"deepseek-v3-2::enabled","modelScoreConfigSlug":"deepseek-v3-2-thinking","modelScoreStateName":"DeepSeek-V3.2 (thinking)","modelScoreStateBestProbability":0.9441666666666667,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":156.6236,"rankLo":146.0,"rankHi":167.0},{"slug":"qwen3-6-35b-a3b","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B","organizationSlug":"alibaba","n":67,"rankEligible":true,"score":147.71,"scoreLo":145.54,"scoreHi":150.13,"modelScoreStateId":"qwen3-6-35b-a3b::default","modelScoreConfigSlug":"qwen-3-6-35b-a3b","modelScoreStateName":"Qwen3.6 35B A3B","modelScoreStateBestProbability":0.8247222222222222,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":157.0481,"rankLo":144.0,"rankHi":170.0},{"slug":"mimo-v2-omni","modelId":"mimo-v2-omni","modelSlug":"mimo-v2-omni","name":"MiMo-V2-Omni","organizationSlug":"xiaomi","n":7,"rankEligible":true,"score":147.54,"scoreLo":144.07,"scoreHi":151.3,"modelScoreStateId":"mimo-v2-omni::enabled","modelScoreConfigSlug":"xiaomi-mimo-v2-omni-thinking","modelScoreStateName":"MiMo-V2-Omni (thinking)","modelScoreStateBestProbability":0.7483333333333333,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":158.6078,"rankLo":126.0,"rankHi":190.0},{"slug":"claude-sonnet-4-5-20250929","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5","organizationSlug":"anthropic","n":20,"rankEligible":true,"score":147.45,"scoreLo":145.29,"scoreHi":149.85,"modelScoreStateId":"claude-sonnet-4-5-20250929::effort:high","modelScoreConfigSlug":"claude-sonnet-4-5-20250929-high","modelScoreStateName":"Sonnet 4.5 (high)","modelScoreStateBestProbability":0.39694444444444443,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":159.8333,"rankLo":146.975,"rankHi":173.0},{"slug":"nemotron-3-ultra-550b-a55b-nvfp4","modelId":"nemotron-3-ultra-550b-a55b-nvfp4","modelSlug":"nemotron-3-ultra-550b-a55b-nvfp4","name":"Nemotron 3 Ultra 550B A55B (NVFP4)","organizationSlug":"nvidia","n":34,"rankEligible":true,"score":147.36,"scoreLo":144.98,"scoreHi":150.1,"modelScoreStateId":"nemotron-3-ultra-550b-a55b-nvfp4::default","modelScoreConfigSlug":"nemotron-3-ultra-550b-a55b-nvfp4","modelScoreStateName":"Nemotron 3 Ultra 550B A55B (NVFP4)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":160.7408,"rankLo":144.0,"rankHi":178.0},{"slug":"qwen3-5-35b-a3b","modelId":"qwen3-5-35b-a3b","modelSlug":"qwen3-5-35b-a3b","name":"Qwen3.5 35B A3B","organizationSlug":"alibaba","n":18,"rankEligible":true,"score":147.23,"scoreLo":145.06,"scoreHi":149.69,"modelScoreStateId":"qwen3-5-35b-a3b::enabled","modelScoreConfigSlug":"qwen3-5-35b-a3b-thinking","modelScoreStateName":"Qwen3.5 35B A3B (thinking)","modelScoreStateBestProbability":0.635,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":162.2286,"rankLo":148.0,"rankHi":176.0},{"slug":"minimax-m2-7","modelId":"minimax-m2-7","modelSlug":"minimax-m2-7","name":"MiniMax M2.7","organizationSlug":"minimax","n":76,"rankEligible":true,"score":147.16,"scoreLo":145.12,"scoreHi":149.54,"modelScoreStateId":"minimax-m2-7::default","modelScoreConfigSlug":"minimax-m2-7","modelScoreStateName":"MiniMax M2.7","modelScoreStateBestProbability":0.9891666666666666,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":163.0692,"rankLo":152.0,"rankHi":174.0},{"slug":"gpt-5-1-codex-mini","modelId":"gpt-5-1-codex-mini","modelSlug":"gpt-5-1-codex-mini","name":"GPT-5.1-Codex mini","organizationSlug":"openai","n":11,"rankEligible":true,"score":147.09,"scoreLo":144.21,"scoreHi":150.27,"modelScoreStateId":"gpt-5-1-codex-mini::effort:high","modelScoreConfigSlug":"gpt-5-1-codex-mini-high","modelScoreStateName":"GPT-5.1-Codex mini (high)","modelScoreStateBestProbability":0.7386111111111111,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":163.5303,"rankLo":138.0,"rankHi":188.0},{"slug":"ring-2-6-1t","modelId":"ring-2-6-1t","modelSlug":"ring-2-6-1t","name":"Ring 2.6 1T","organizationSlug":"inclusion-ai","n":18,"rankEligible":true,"score":146.87,"scoreLo":144.1,"scoreHi":149.85,"modelScoreStateId":"ring-2-6-1t::default","modelScoreConfigSlug":"ring-2-6-1t","modelScoreStateName":"Ring 2.6 1T","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":166.0244,"rankLo":143.0,"rankHi":188.0},{"slug":"qwen3-7-flash","modelId":"qwen3-7-flash","modelSlug":"qwen3-7-flash","name":"Qwen3.7 Flash","organizationSlug":"alibaba","n":11,"rankEligible":true,"score":146.76,"scoreLo":143.41,"scoreHi":150.33,"modelScoreStateId":"qwen3-7-flash::default","modelScoreConfigSlug":"qwen3-7-flash","modelScoreStateName":"Qwen3.7 Flash","modelScoreStateBestProbability":0.9658333333333333,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":167.2039,"rankLo":136.0,"rankHi":198.0},{"slug":"mimo-v2-pro","modelId":"mimo-v2-pro","modelSlug":"mimo-v2-pro","name":"MiMo-V2-Pro","organizationSlug":"xiaomi","n":8,"rankEligible":true,"score":146.68,"scoreLo":143.65,"scoreHi":149.85,"modelScoreStateId":"mimo-v2-pro::enabled","modelScoreConfigSlug":"xiaomi-mimo-v2-pro-thinking","modelScoreStateName":"MiMo-V2-Pro (thinking)","modelScoreStateBestProbability":0.6516666666666666,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":168.0267,"rankLo":142.0,"rankHi":194.0},{"slug":"a-x-k2","modelId":"a-x-k2","modelSlug":"a-x-k2","name":"A.X-K2","organizationSlug":"sk-telecom","n":9,"rankEligible":true,"score":146.62,"scoreLo":143.24,"scoreHi":150.11,"modelScoreStateId":"a-x-k2::enabled","modelScoreConfigSlug":"a-x-k2-thinking","modelScoreStateName":"A.X-K2 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":168.6208,"rankLo":138.0,"rankHi":201.0},{"slug":"gemini-robotics-er-1-5-preview","modelId":"gemini-robotics-er-1-5-preview","modelSlug":"gemini-robotics-er-1-5-preview","name":"Gemini Robotics-ER 1.5","organizationSlug":"google","n":4,"rankEligible":true,"score":146.5,"scoreLo":142.48,"scoreHi":150.62,"modelScoreStateId":"gemini-robotics-er-1-5-preview::enabled","modelScoreConfigSlug":"gemini-robotics-er-1-5-preview-thinking","modelScoreStateName":"Gemini Robotics-ER 1.5 (thinking)","modelScoreStateBestProbability":0.9122222222222223,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":169.9358,"rankLo":130.0,"rankHi":209.0},{"slug":"qwen3-5-omni-plus","modelId":"qwen3-5-omni-plus","modelSlug":"qwen3-5-omni-plus","name":"Qwen3.5-Omni-Plus","organizationSlug":"alibaba","n":42,"rankEligible":true,"score":146.46,"scoreLo":144.0,"scoreHi":149.12,"modelScoreStateId":"qwen3-5-omni-plus::default","modelScoreConfigSlug":"qwen3-5-omni-plus","modelScoreStateName":"Qwen3.5-Omni-Plus","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":170.4789,"rankLo":151.975,"rankHi":189.0},{"slug":"mimo-v2-flash","modelId":"mimo-v2-flash","modelSlug":"mimo-v2-flash","name":"MiMo-V2-Flash","organizationSlug":"xiaomi","n":11,"rankEligible":true,"score":146.45,"scoreLo":143.39,"scoreHi":149.8,"modelScoreStateId":"mimo-v2-flash::enabled","modelScoreConfigSlug":"mimo-v2-flash-thinking","modelScoreStateName":"MiMo-V2-Flash (thinking)","modelScoreStateBestProbability":0.9805555555555555,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":170.4128,"rankLo":142.0,"rankHi":198.0},{"slug":"longcat-2-0","modelId":"longcat-2-0","modelSlug":"longcat-2-0","name":"LongCat-2.0","organizationSlug":"meituan","n":13,"rankEligible":true,"score":146.37,"scoreLo":143.41,"scoreHi":149.47,"modelScoreStateId":"longcat-2-0::default","modelScoreConfigSlug":"longcat-2-0","modelScoreStateName":"LongCat-2.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":171.3269,"rankLo":146.0,"rankHi":198.0},{"slug":"gemini-2-5-pro","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro","organizationSlug":"google","n":18,"rankEligible":true,"score":146.22,"scoreLo":144.14,"scoreHi":148.54,"modelScoreStateId":"gemini-2-5-pro::effort:high","modelScoreConfigSlug":"gemini-2-5-pro-high","modelScoreStateName":"Gemini 2.5 Pro (high)","modelScoreStateBestProbability":0.9255555555555556,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":173.0119,"rankLo":161.0,"rankHi":185.0},{"slug":"gemini-2-5-pro-exp-03-25","modelId":"gemini-2-5-pro-exp-03-25","modelSlug":"gemini-2-5-pro-exp-03-25","name":"Gemini 2.5 Pro Experimental 03-25","organizationSlug":"google","n":27,"rankEligible":true,"score":146.06,"scoreLo":143.2,"scoreHi":149.28,"modelScoreStateId":"gemini-2-5-pro-exp-03-25::default","modelScoreConfigSlug":"gemini-2-5-pro-exp-03-25","modelScoreStateName":"Gemini 2.5 Pro Experimental 03-25","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":174.7042,"rankLo":148.0,"rankHi":199.0},{"slug":"grok-4-0709","modelId":"grok-4-0709","modelSlug":"grok-4-0709","name":"Grok 4","organizationSlug":"xai","n":97,"rankEligible":true,"score":146.06,"scoreLo":144.13,"scoreHi":148.34,"modelScoreStateId":"grok-4-0709::default","modelScoreConfigSlug":"grok-4-0709","modelScoreStateName":"Grok 4","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":174.6767,"rankLo":164.0,"rankHi":186.0},{"slug":"claude-opus-4-1-20250805","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1","organizationSlug":"anthropic","n":42,"rankEligible":true,"score":145.99,"scoreLo":143.85,"scoreHi":148.31,"modelScoreStateId":"claude-opus-4-1-20250805::default","modelScoreConfigSlug":"claude-opus-4-1-20250805","modelScoreStateName":"Opus 4.1","modelScoreStateBestProbability":0.5041666666666667,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":175.5003,"rankLo":162.0,"rankHi":189.0},{"slug":"k2-horizon-7b","modelId":"k2-horizon-7b","modelSlug":"k2-horizon-7b","name":"K2 Horizon 7B","organizationSlug":"llm360","n":11,"rankEligible":true,"score":145.77,"scoreLo":142.82,"scoreHi":149.06,"modelScoreStateId":"k2-horizon-7b::enabled","modelScoreConfigSlug":"k2-horizon-7b-thinking","modelScoreStateName":"K2 Horizon 7B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":177.7975,"rankLo":152.0,"rankHi":205.0},{"slug":"minimax-m2-5","modelId":"minimax-m2-5","modelSlug":"minimax-m2-5","name":"MiniMax M2.5","organizationSlug":"minimax","n":44,"rankEligible":true,"score":145.75,"scoreLo":143.62,"scoreHi":148.11,"modelScoreStateId":"minimax-m2-5::default","modelScoreConfigSlug":"minimax-m2-5","modelScoreStateName":"MiniMax M2.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":178.1172,"rankLo":165.0,"rankHi":192.0},{"slug":"gpt-5-mini","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini","organizationSlug":"openai","n":112,"rankEligible":true,"score":145.71,"scoreLo":143.84,"scoreHi":147.83,"modelScoreStateId":"gpt-5-mini::effort:high","modelScoreConfigSlug":"gpt-5-mini-high","modelScoreStateName":"GPT-5 Mini (high)","modelScoreStateBestProbability":0.6261111111111111,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":178.5128,"rankLo":170.0,"rankHi":188.0},{"slug":"gemini-3-1-flash-lite","modelId":"gemini-3-1-flash-lite","modelSlug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite","organizationSlug":"google","n":15,"rankEligible":true,"score":145.63,"scoreLo":143.25,"scoreHi":148.23,"modelScoreStateId":"gemini-3-1-flash-lite::effort:high","modelScoreConfigSlug":"gemini-3-1-flash-lite-high","modelScoreStateName":"Gemini 3.1 Flash-Lite (high)","modelScoreStateBestProbability":0.7658333333333334,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":179.4025,"rankLo":161.0,"rankHi":198.0},{"slug":"kimi-k2-thinking","modelId":"kimi-k2-thinking","modelSlug":"kimi-k2-thinking","name":"Kimi K2 Thinking","organizationSlug":"moonshot-ai","n":86,"rankEligible":true,"score":145.56,"scoreLo":143.56,"scoreHi":147.82,"modelScoreStateId":"kimi-k2-thinking::default","modelScoreConfigSlug":"kimi-k2-thinking","modelScoreStateName":"Kimi K2 Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":180.0411,"rankLo":168.0,"rankHi":192.0},{"slug":"gemini-3-1-flash-lite-preview","modelId":"gemini-3-1-flash-lite-preview","modelSlug":"gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash-Lite Preview","organizationSlug":"google","n":15,"rankEligible":true,"score":145.5,"scoreLo":143.02,"scoreHi":148.11,"modelScoreStateId":"gemini-3-1-flash-lite-preview::enabled","modelScoreConfigSlug":"gemini-3-1-flash-lite-preview-thinking","modelScoreStateName":"Gemini 3.1 Flash-Lite Preview (thinking)","modelScoreStateBestProbability":0.7616666666666667,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":180.7361,"rankLo":161.0,"rankHi":200.0},{"slug":"deepseek-v3-2-exp","modelId":"deepseek-v3-2-exp","modelSlug":"deepseek-v3-2-exp","name":"DeepSeek-V3.2-Exp","organizationSlug":"deepseek","n":23,"rankEligible":true,"score":145.41,"scoreLo":143.26,"scoreHi":147.8,"modelScoreStateId":"deepseek-v3-2-exp::enabled","modelScoreConfigSlug":"deepseek-v3-2-exp-thinking","modelScoreStateName":"DeepSeek-V3.2-Exp (thinking)","modelScoreStateBestProbability":0.9758333333333333,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":181.7181,"rankLo":167.0,"rankHi":197.0},{"slug":"ernie-5-1","modelId":"ernie-5-1","modelSlug":"ernie-5-1","name":"ERNIE 5.1","organizationSlug":"baidu","n":8,"rankEligible":true,"score":145.4,"scoreLo":141.25,"scoreHi":149.75,"modelScoreStateId":"ernie-5-1::default","modelScoreConfigSlug":"ernie-5-1","modelScoreStateName":"ERNIE 5.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":181.5572,"rankLo":140.0,"rankHi":219.0},{"slug":"deepseek-v3-1-terminus","modelId":"deepseek-v3-1-terminus","modelSlug":"deepseek-v3-1-terminus","name":"DeepSeek-V3.1-Terminus","organizationSlug":"deepseek","n":24,"rankEligible":true,"score":145.39,"scoreLo":143.11,"scoreHi":147.89,"modelScoreStateId":"deepseek-v3-1-terminus::enabled","modelScoreConfigSlug":"deepseek-v3-1-terminus-thinking","modelScoreStateName":"DeepSeek-V3.1-Terminus (thinking)","modelScoreStateBestProbability":0.9375,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":181.9353,"rankLo":165.0,"rankHi":200.0},{"slug":"grok-4-1-fast","modelId":"grok-4-1-fast","modelSlug":"grok-4-1-fast","name":"Grok 4.1 Fast","organizationSlug":"xai","n":83,"rankEligible":true,"score":144.71,"scoreLo":142.77,"scoreHi":146.87,"modelScoreStateId":"grok-4-1-fast::enabled","modelScoreConfigSlug":"grok-4-1-fast-thinking","modelScoreStateName":"Grok 4.1 Fast (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":189.0908,"rankLo":178.0,"rankHi":201.0},{"slug":"glm-4-6","modelId":"glm-4-6","modelSlug":"glm-4-6","name":"GLM 4.6","organizationSlug":"zhipu-ai","n":19,"rankEligible":true,"score":144.5,"scoreLo":142.36,"scoreHi":146.93,"modelScoreStateId":"glm-4-6::enabled","modelScoreConfigSlug":"glm-4-6-thinking","modelScoreStateName":"GLM 4.6 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":191.4594,"rankLo":176.0,"rankHi":208.0},{"slug":"claude-opus-4-20250514","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4","organizationSlug":"anthropic","n":5,"rankEligible":true,"score":144.47,"scoreLo":141.47,"scoreHi":147.89,"modelScoreStateId":"claude-opus-4-20250514::effort:10k","modelScoreConfigSlug":"claude-opus-4-20250514-10k-helm-capabilities","modelScoreStateName":"Opus 4 (10k)","modelScoreStateBestProbability":0.4672222222222222,"modelScoreReasoningCondition":"effort:10k","modelScoreReasoningEffort":"10k","modelScoreHarness":"none","rankMean":191.5094,"rankLo":161.0,"rankHi":217.0},{"slug":"ernie-5-0-thinking-preview","modelId":"ernie-5-0-thinking-preview","modelSlug":"ernie-5-0-thinking-preview","name":"ERNIE 5.0 Thinking Preview","organizationSlug":"baidu","n":8,"rankEligible":true,"score":144.44,"scoreLo":140.63,"scoreHi":148.5,"modelScoreStateId":"ernie-5-0-thinking-preview::default","modelScoreConfigSlug":"ernie-5-0-thinking-preview","modelScoreStateName":"ERNIE 5.0 Thinking Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":191.6656,"rankLo":155.0,"rankHi":228.0},{"slug":"grok-4-1","modelId":"grok-4-1","modelSlug":"grok-4-1","name":"Grok 4.1","organizationSlug":"xai","n":4,"rankEligible":true,"score":144.42,"scoreLo":139.75,"scoreHi":149.24,"modelScoreStateId":"grok-4-1::enabled","modelScoreConfigSlug":"grok-4-1-thinking","modelScoreStateName":"Grok 4.1 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":191.7581,"rankLo":144.0,"rankHi":242.0},{"slug":"mistral-medium-3-5-128b","modelId":"mistral-medium-3-5-128b","modelSlug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5","organizationSlug":"mistral-ai","n":50,"rankEligible":true,"score":144.32,"scoreLo":142.22,"scoreHi":146.63,"modelScoreStateId":"mistral-medium-3-5-128b::effort:high","modelScoreConfigSlug":"mistral-medium-3-5-128b-high","modelScoreStateName":"Mistral Medium 3.5 (high)","modelScoreStateBestProbability":0.8738888888888889,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":193.37,"rankLo":179.0,"rankHi":208.0},{"slug":"minimax-m2-1","modelId":"minimax-m2-1","modelSlug":"minimax-m2-1","name":"MiniMax M2.1","organizationSlug":"minimax","n":35,"rankEligible":true,"score":144.3,"scoreLo":141.97,"scoreHi":146.86,"modelScoreStateId":"minimax-m2-1::default","modelScoreConfigSlug":"minimax-m2-1","modelScoreStateName":"MiniMax M2.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":193.4181,"rankLo":174.0,"rankHi":211.0},{"slug":"claude-sonnet-4-20250514","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4","organizationSlug":"anthropic","n":11,"rankEligible":true,"score":144.2,"scoreLo":142.3,"scoreHi":146.25,"modelScoreStateId":"claude-sonnet-4-20250514::effort:16k","modelScoreConfigSlug":"claude-sonnet-4-20250514-16k","modelScoreStateName":"Sonnet 4 (16k)","modelScoreStateBestProbability":0.4263888888888889,"modelScoreReasoningCondition":"effort:16k","modelScoreReasoningEffort":"16k","modelScoreHarness":"none","rankMean":194.6669,"rankLo":183.0,"rankHi":207.0},{"slug":"gemini-2-5-pro-preview-03-25","modelId":"gemini-2-5-pro-preview-03-25","modelSlug":"gemini-2-5-pro-preview-03-25","name":"Gemini 2.5 Pro Preview 03-25","organizationSlug":"google","n":27,"rankEligible":true,"score":144.13,"scoreLo":141.73,"scoreHi":146.64,"modelScoreStateId":"gemini-2-5-pro-preview-03-25::default","modelScoreConfigSlug":"gemini-2-5-pro-preview-03-25","modelScoreStateName":"Gemini 2.5 Pro Preview 03-25","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":195.1753,"rankLo":175.0,"rankHi":214.0},{"slug":"doubao-seed-code-preview-251028","modelId":"doubao-seed-code-preview-251028","modelSlug":"doubao-seed-code-preview-251028","name":"Doubao-Seed-Code","organizationSlug":"bytedance","n":9,"rankEligible":true,"score":144.1,"scoreLo":140.51,"scoreHi":147.82,"modelScoreStateId":"doubao-seed-code-preview-251028::enabled","modelScoreConfigSlug":"doubao-seed-code-preview-251028-thinking","modelScoreStateName":"Doubao-Seed-Code (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":195.2178,"rankLo":162.0,"rankHi":228.025},{"slug":"gemma-4-26b-a4b-it","modelId":"gemma-4-26b-a4b-it","modelSlug":"gemma-4-26b-a4b-it","name":"Gemma 4 26B A4B IT","organizationSlug":"google","n":35,"rankEligible":true,"score":144.06,"scoreLo":142.14,"scoreHi":146.27,"modelScoreStateId":"gemma-4-26b-a4b-it::enabled","modelScoreConfigSlug":"gemma-4-26b-a4b-it-thinking","modelScoreStateName":"Gemma 4 26B A4B IT (thinking)","modelScoreStateBestProbability":0.9902777777777778,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":196.1433,"rankLo":183.0,"rankHi":209.0},{"slug":"nemotron-3-super-120b-a12b-bf16","modelId":"nemotron-3-super-120b-a12b-bf16","modelSlug":"nemotron-3-super-120b-a12b-bf16","name":"Nemotron 3 Super 120B A12B","organizationSlug":"nvidia","n":23,"rankEligible":true,"score":144.05,"scoreLo":141.91,"scoreHi":146.33,"modelScoreStateId":"nemotron-3-super-120b-a12b-bf16::enabled","modelScoreConfigSlug":"nemotron-3-super-120b-a12b-bf16-thinking","modelScoreStateName":"Nemotron 3 Super 120B A12B (thinking)","modelScoreStateBestProbability":0.9994444444444445,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":196.1117,"rankLo":179.0,"rankHi":211.025},{"slug":"o4-mini","modelId":"o4-mini","modelSlug":"o4-mini","name":"o4-mini","organizationSlug":"openai","n":83,"rankEligible":true,"score":143.86,"scoreLo":142.05,"scoreHi":145.9,"modelScoreStateId":"o4-mini::effort:high","modelScoreConfigSlug":"o4-mini-high","modelScoreStateName":"o4-mini (high)","modelScoreStateBestProbability":0.7908333333333334,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":198.2953,"rankLo":187.0,"rankHi":209.0},{"slug":"g9v3-39a5b","modelId":"g9v3-39a5b","modelSlug":"g9v3-39a5b","name":"G9v3-39A5B","organizationSlug":"ai9stars","n":5,"rankEligible":true,"score":143.82,"scoreLo":139.99,"scoreHi":147.84,"modelScoreStateId":"g9v3-39a5b::enabled","modelScoreConfigSlug":"g9v3-39a5b-thinking","modelScoreStateName":"G9v3-39A5B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":198.2647,"rankLo":162.0,"rankHi":239.0},{"slug":"claude-haiku-4-5-20251001","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5","organizationSlug":"anthropic","n":10,"rankEligible":true,"score":143.77,"scoreLo":141.34,"scoreHi":146.34,"modelScoreStateId":"claude-haiku-4-5-20251001::effort:high","modelScoreConfigSlug":"claude-haiku-4-5-20251001-high","modelScoreStateName":"Haiku 4.5 (high)","modelScoreStateBestProbability":0.5319444444444444,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":198.9622,"rankLo":178.0,"rankHi":217.0},{"slug":"grok-4-fast","modelId":"grok-4-fast","modelSlug":"grok-4-fast","name":"Grok 4 Fast","organizationSlug":"xai","n":63,"rankEligible":true,"score":143.77,"scoreLo":141.72,"scoreHi":145.99,"modelScoreStateId":"grok-4-fast::enabled","modelScoreConfigSlug":"grok-4-fast-thinking","modelScoreStateName":"Grok 4 Fast (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":199.0608,"rankLo":185.0,"rankHi":212.0},{"slug":"deepseek-v3-1","modelId":"deepseek-v3-1","modelSlug":"deepseek-v3-1","name":"DeepSeek-V3.1","organizationSlug":"deepseek","n":29,"rankEligible":true,"score":143.69,"scoreLo":141.81,"scoreHi":145.76,"modelScoreStateId":"deepseek-v3-1::enabled","modelScoreConfigSlug":"deepseek-v3-1-thinking","modelScoreStateName":"DeepSeek-V3.1 (thinking)","modelScoreStateBestProbability":0.9983333333333333,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":200.0289,"rankLo":187.0,"rankHi":211.0},{"slug":"qwen3-max-2025-09-23","modelId":"qwen3-max-2025-09-23","modelSlug":"qwen3-max-2025-09-23","name":"Qwen3-Max (2025-09-23)","organizationSlug":"alibaba","n":21,"rankEligible":true,"score":143.64,"scoreLo":141.06,"scoreHi":146.48,"modelScoreStateId":"qwen3-max-2025-09-23::default","modelScoreConfigSlug":"qwen3-max-2025-09-23","modelScoreStateName":"Qwen3-Max (2025-09-23)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":200.2242,"rankLo":178.0,"rankHi":220.0},{"slug":"gemini-2-5-pro-preview-05-06","modelId":"gemini-2-5-pro-preview-05-06","modelSlug":"gemini-2-5-pro-preview-05-06","name":"Gemini 2.5 Pro Preview 05-06","organizationSlug":"google","n":20,"rankEligible":true,"score":143.49,"scoreLo":140.71,"scoreHi":146.55,"modelScoreStateId":"gemini-2-5-pro-preview-05-06::default","modelScoreConfigSlug":"gemini-2-5-pro-may-25","modelScoreStateName":"Gemini 2.5 Pro Preview 05-06","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":201.5911,"rankLo":176.0,"rankHi":225.0},{"slug":"gemini-2-5-flash-preview-09-2025","modelId":"gemini-2-5-flash-preview-09-2025","modelSlug":"gemini-2-5-flash-preview-09-2025","name":"Gemini 2.5 Flash Preview (09-2025)","organizationSlug":"google","n":31,"rankEligible":true,"score":143.43,"scoreLo":141.17,"scoreHi":145.96,"modelScoreStateId":"gemini-2-5-flash-preview-09-2025::enabled","modelScoreConfigSlug":"gemini-2-5-flash-sep-25-thinking","modelScoreStateName":"Gemini 2.5 Flash Preview (09-2025) (thinking)","modelScoreStateBestProbability":0.5497222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":202.3253,"rankLo":185.0,"rankHi":219.0},{"slug":"minimax-m2","modelId":"minimax-m2","modelSlug":"minimax-m2","name":"MiniMax M2","organizationSlug":"minimax","n":31,"rankEligible":true,"score":143.4,"scoreLo":141.11,"scoreHi":145.81,"modelScoreStateId":"minimax-m2::default","modelScoreConfigSlug":"minimax-m2","modelScoreStateName":"MiniMax M2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":202.7456,"rankLo":185.0,"rankHi":219.0},{"slug":"solar-pro4","modelId":"solar-pro4","modelSlug":"solar-pro4","name":"Solar Pro 4","organizationSlug":"upstage","n":7,"rankEligible":true,"score":143.14,"scoreLo":138.87,"scoreHi":147.53,"modelScoreStateId":"solar-pro4::default","modelScoreConfigSlug":"solar-pro4","modelScoreStateName":"Solar Pro 4","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":205.3372,"rankLo":165.0,"rankHi":249.025},{"slug":"qwen3-vl-235b-a22b-thinking","modelId":"qwen3-vl-235b-a22b-thinking","modelSlug":"qwen3-vl-235b-a22b-thinking","name":"Qwen3 VL 235B A22B Thinking","organizationSlug":"alibaba","n":64,"rankEligible":true,"score":142.95,"scoreLo":140.76,"scoreHi":145.39,"modelScoreStateId":"qwen3-vl-235b-a22b-thinking::default","modelScoreConfigSlug":"qwen3-vl-235b-a22b-thinking","modelScoreStateName":"Qwen3 VL 235B A22B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":207.1186,"rankLo":191.0,"rankHi":222.0},{"slug":"qwen3-vl-plus-2025-09-23","modelId":"qwen3-vl-plus-2025-09-23","modelSlug":"qwen3-vl-plus-2025-09-23","name":"Qwen3-VL-Plus-2025-09-23","organizationSlug":"alibaba","n":4,"rankEligible":true,"score":142.74,"scoreLo":133.96,"scoreHi":151.63,"modelScoreStateId":"qwen3-vl-plus-2025-09-23::default","modelScoreConfigSlug":"qwen3-vl-plus-2025-09-23","modelScoreStateName":"Qwen3-VL-Plus-2025-09-23","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":211.88,"rankLo":115.975,"rankHi":321.025},{"slug":"qwen3-5-9b","modelId":"qwen3-5-9b","modelSlug":"qwen3-5-9b","name":"Qwen3.5-9B","organizationSlug":"alibaba","n":21,"rankEligible":true,"score":142.66,"scoreLo":140.79,"scoreHi":144.78,"modelScoreStateId":"qwen3-5-9b::enabled","modelScoreConfigSlug":"qwen3-5-9b-thinking","modelScoreStateName":"Qwen3.5-9B (thinking)","modelScoreStateBestProbability":0.7311111111111112,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":209.9106,"rankLo":198.0,"rankHi":222.0},{"slug":"qwen3-235b-a22b-thinking-2507","modelId":"qwen3-235b-a22b-thinking-2507","modelSlug":"qwen3-235b-a22b-thinking-2507","name":"Qwen3 235B A22B Thinking 2507","organizationSlug":"alibaba","n":58,"rankEligible":true,"score":142.64,"scoreLo":140.6,"scoreHi":144.74,"modelScoreStateId":"qwen3-235b-a22b-thinking-2507::default","modelScoreConfigSlug":"qwen3-235b-a22b-2507-thinking","modelScoreStateName":"Qwen3 235B A22B Thinking 2507","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":210.0811,"rankLo":197.0,"rankHi":223.0},{"slug":"k-exaone-236b-a23b","modelId":"k-exaone-236b-a23b","modelSlug":"k-exaone-236b-a23b","name":"K-EXAONE 236B-A23B","organizationSlug":"lg-ai-research","n":11,"rankEligible":true,"score":142.11,"scoreLo":139.13,"scoreHi":145.12,"modelScoreStateId":"k-exaone-236b-a23b::enabled","modelScoreConfigSlug":"k-exaone-236b-a23b-thinking","modelScoreStateName":"K-EXAONE 236B-A23B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":215.5597,"rankLo":189.975,"rankHi":245.0},{"slug":"gemini-2-5-flash","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash","organizationSlug":"google","n":8,"rankEligible":true,"score":142.05,"scoreLo":139.95,"scoreHi":144.17,"modelScoreStateId":"gemini-2-5-flash::effort:high","modelScoreConfigSlug":"gemini-2-5-flash-high","modelScoreStateName":"Gemini 2.5 Flash (high)","modelScoreStateBestProbability":0.8408333333333333,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":215.5917,"rankLo":203.0,"rankHi":232.0},{"slug":"qwen3-max-preview","modelId":"qwen3-max-preview","modelSlug":"qwen3-max-preview","name":"Qwen3 Max Preview","organizationSlug":"alibaba","n":11,"rankEligible":true,"score":142.04,"scoreLo":139.05,"scoreHi":145.27,"modelScoreStateId":"qwen3-max-preview::default","modelScoreConfigSlug":"qwen3-max-preview","modelScoreStateName":"Qwen3 Max Preview","modelScoreStateBestProbability":0.5580555555555555,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":216.4169,"rankLo":188.0,"rankHi":247.025},{"slug":"nova-2-pro-preview-20251202-v1-0","modelId":"nova-2-pro-preview-20251202-v1-0","modelSlug":"nova-2-pro-preview-20251202-v1-0","name":"Nova 2 Pro (Preview)","organizationSlug":"amazon","n":12,"rankEligible":true,"score":142.01,"scoreLo":139.41,"scoreHi":144.82,"modelScoreStateId":"nova-2-pro-preview-20251202-v1-0::effort:medium","modelScoreConfigSlug":"nova-2-pro-preview-20251202-v1-0-medium","modelScoreStateName":"Nova 2 Pro (Preview) (medium)","modelScoreStateBestProbability":0.7711111111111111,"modelScoreReasoningCondition":"effort:medium","modelScoreReasoningEffort":"medium","modelScoreHarness":"none","rankMean":216.5375,"rankLo":194.0,"rankHi":243.0},{"slug":"k2-horizon-3-7b","modelId":"k2-horizon-3-7b","modelSlug":"k2-horizon-3-7b","name":"K2 Horizon 3.7B","organizationSlug":"llm360","n":10,"rankEligible":true,"score":141.71,"scoreLo":137.62,"scoreHi":145.62,"modelScoreStateId":"k2-horizon-3-7b::enabled","modelScoreConfigSlug":"k2-horizon-3-7b-thinking","modelScoreStateName":"K2 Horizon 3.7B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":220.7464,"rankLo":184.0,"rankHi":269.025},{"slug":"qwen3-vl-235b-a22b-instruct","modelId":"qwen3-vl-235b-a22b-instruct","modelSlug":"qwen3-vl-235b-a22b-instruct","name":"Qwen3-VL-235B-A22B-Instruct","organizationSlug":"alibaba","n":62,"rankEligible":true,"score":141.64,"scoreLo":139.63,"scoreHi":143.84,"modelScoreStateId":"qwen3-vl-235b-a22b-instruct::default","modelScoreConfigSlug":"qwen3-vl-235b-a22b","modelScoreStateName":"Qwen3-VL-235B-A22B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":219.8764,"rankLo":205.0,"rankHi":238.0},{"slug":"exaone-4-5-33b","modelId":"exaone-4-5-33b","modelSlug":"exaone-4-5-33b","name":"EXAONE 4.5 33B","organizationSlug":"lg-ai-research","n":9,"rankEligible":true,"score":141.43,"scoreLo":138.15,"scoreHi":144.95,"modelScoreStateId":"exaone-4-5-33b::enabled","modelScoreConfigSlug":"exaone-4-5-33b-thinking","modelScoreStateName":"EXAONE 4.5 33B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":223.3822,"rankLo":193.0,"rankHi":263.0},{"slug":"o1","modelId":"o1","modelSlug":"o1","name":"o1 (2024-12-17)","organizationSlug":"openai","n":35,"rankEligible":true,"score":141.28,"scoreLo":139.39,"scoreHi":143.36,"modelScoreStateId":"o1::effort:high","modelScoreConfigSlug":"o1-high","modelScoreStateName":"o1 (2024-12-17) (high)","modelScoreStateBestProbability":0.5569444444444445,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":223.5944,"rankLo":210.0,"rankHi":241.0},{"slug":"qwen3-coder-480b-a35b-instruct","modelId":"qwen3-coder-480b-a35b-instruct","modelSlug":"qwen3-coder-480b-a35b-instruct","name":"Qwen3-Coder-480B-A35B-Instruct","organizationSlug":"alibaba","n":33,"rankEligible":true,"score":141.18,"scoreLo":138.91,"scoreHi":143.53,"modelScoreStateId":"qwen3-coder-480b-a35b-instruct::default","modelScoreConfigSlug":"qwen3-coder-480b-a35b-instruct","modelScoreStateName":"Qwen3-Coder-480B-A35B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":225.1411,"rankLo":206.0,"rankHi":249.0},{"slug":"mercury-2-5","modelId":"mercury-2-5","modelSlug":"mercury-2-5","name":"Mercury 2.5","organizationSlug":"inception-labs","n":7,"rankEligible":true,"score":141.08,"scoreLo":137.71,"scoreHi":144.59,"modelScoreStateId":"mercury-2-5::enabled","modelScoreConfigSlug":"mercury-2-5-thinking","modelScoreStateName":"Mercury 2.5 (thinking)","modelScoreStateBestProbability":0.7108333333333333,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":227.4853,"rankLo":195.0,"rankHi":268.0},{"slug":"qwen3-5-4b","modelId":"qwen3-5-4b","modelSlug":"qwen3-5-4b","name":"Qwen3.5-4B","organizationSlug":"alibaba","n":58,"rankEligible":true,"score":140.98,"scoreLo":139.19,"scoreHi":142.96,"modelScoreStateId":"qwen3-5-4b::enabled","modelScoreConfigSlug":"qwen3-5-4b-thinking","modelScoreStateName":"Qwen3.5-4B (thinking)","modelScoreStateBestProbability":0.9972222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":226.9161,"rankLo":214.0,"rankHi":244.0},{"slug":"apriel-1-6-15b-thinker","modelId":"apriel-1-6-15b-thinker","modelSlug":"apriel-1-6-15b-thinker","name":"Apriel-1.6-15B-Thinker","organizationSlug":"servicenow","n":6,"rankEligible":true,"score":140.93,"scoreLo":136.32,"scoreHi":145.36,"modelScoreStateId":"apriel-1-6-15b-thinker::default","modelScoreConfigSlug":"apriel-1-6-15b-thinker","modelScoreStateName":"Apriel-1.6-15B-Thinker","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":230.1147,"rankLo":185.0,"rankHi":287.0},{"slug":"qwen3-coder-next","modelId":"qwen3-coder-next","modelSlug":"qwen3-coder-next","name":"Qwen3-Coder-Next","organizationSlug":"alibaba","n":32,"rankEligible":true,"score":140.9,"scoreLo":138.77,"scoreHi":143.19,"modelScoreStateId":"qwen3-coder-next::default","modelScoreConfigSlug":"qwen3-coder-next","modelScoreStateName":"Qwen3-Coder-Next","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":228.3503,"rankLo":210.0,"rankHi":251.0},{"slug":"grok-3-mini","modelId":"grok-3-mini","modelSlug":"grok-3-mini","name":"Grok 3 Mini","organizationSlug":"xai","n":26,"rankEligible":true,"score":140.81,"scoreLo":138.27,"scoreHi":143.28,"modelScoreStateId":"grok-3-mini::effort:high","modelScoreConfigSlug":"grok-3-mini-high","modelScoreStateName":"Grok 3 Mini (high)","modelScoreStateBestProbability":0.9713888888888889,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":229.6803,"rankLo":209.0,"rankHi":257.0},{"slug":"gpt-oss-120b","modelId":"gpt-oss-120b","modelSlug":"gpt-oss-120b","name":"gpt-oss-120b","organizationSlug":"openai","n":66,"rankEligible":true,"score":140.76,"scoreLo":139.16,"scoreHi":142.48,"modelScoreStateId":"gpt-oss-120b::effort:high","modelScoreConfigSlug":"gpt-oss-120b-high","modelScoreStateName":"gpt-oss-120b (high)","modelScoreStateBestProbability":0.9911111111111112,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":229.5903,"rankLo":218.0,"rankHi":242.0},{"slug":"claude-3-7-sonnet-20250219","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet","organizationSlug":"anthropic","n":8,"rankEligible":true,"score":140.74,"scoreLo":138.89,"scoreHi":142.69,"modelScoreStateId":"claude-3-7-sonnet-20250219::effort:16k","modelScoreConfigSlug":"claude-3-7-sonnet-20250219-16k","modelScoreStateName":"Claude 3.7 Sonnet (16k)","modelScoreStateBestProbability":0.7458333333333333,"modelScoreReasoningCondition":"effort:16k","modelScoreReasoningEffort":"16k","modelScoreHarness":"none","rankMean":230.0469,"rankLo":215.0,"rankHi":249.0},{"slug":"glm-4-5","modelId":"glm-4-5","modelSlug":"glm-4-5","name":"GLM-4.5","organizationSlug":"zhipu-ai","n":15,"rankEligible":true,"score":140.74,"scoreLo":138.48,"scoreHi":143.09,"modelScoreStateId":"glm-4-5::enabled","modelScoreConfigSlug":"glm-4-5-thinking","modelScoreStateName":"GLM-4.5 (thinking)","modelScoreStateBestProbability":0.9991666666666666,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":230.3597,"rankLo":211.0,"rankHi":254.0},{"slug":"gemma-4-12b-it","modelId":"gemma-4-12b-it","modelSlug":"gemma-4-12b-it","name":"Gemma 4 12B IT","organizationSlug":"google","n":69,"rankEligible":true,"score":140.72,"scoreLo":138.59,"scoreHi":142.82,"modelScoreStateId":"gemma-4-12b-it::enabled","modelScoreConfigSlug":"gemma-4-12b-it-thinking","modelScoreStateName":"Gemma 4 12B IT (thinking)","modelScoreStateBestProbability":0.9972222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":230.5536,"rankLo":213.0,"rankHi":252.0},{"slug":"qwen3-235b-a22b-instruct-2507","modelId":"qwen3-235b-a22b-instruct-2507","modelSlug":"qwen3-235b-a22b-instruct-2507","name":"Qwen3 235B A22B Instruct 2507","organizationSlug":"alibaba","n":54,"rankEligible":true,"score":140.64,"scoreLo":138.72,"scoreHi":142.61,"modelScoreStateId":"qwen3-235b-a22b-instruct-2507::default","modelScoreConfigSlug":"qwen3-235b-a22b-2507","modelScoreStateName":"Qwen3 235B A22B Instruct 2507","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":231.4183,"rankLo":216.0,"rankHi":250.0},{"slug":"kat-coder-pro","modelId":"kat-coder-pro","modelSlug":"kat-coder-pro","name":"KAT-Coder-Pro V1","organizationSlug":"kuaishou","n":11,"rankEligible":true,"score":140.63,"scoreLo":137.15,"scoreHi":144.23,"modelScoreStateId":"kat-coder-pro::default","modelScoreConfigSlug":"kat-coder-pro","modelScoreStateName":"KAT-Coder-Pro V1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":232.9656,"rankLo":198.0,"rankHi":276.0},{"slug":"o1-pro","modelId":"o1-pro","modelSlug":"o1-pro","name":"o1-pro","organizationSlug":"openai","n":8,"rankEligible":true,"score":140.59,"scoreLo":137.26,"scoreHi":144.11,"modelScoreStateId":"o1-pro::default","modelScoreConfigSlug":"o1-pro","modelScoreStateName":"o1-pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":233.2267,"rankLo":200.0,"rankHi":274.0},{"slug":"north-mini-code-1-0","modelId":"north-mini-code-1-0","modelSlug":"north-mini-code-1-0","name":"North Mini Code","organizationSlug":"cohere","n":15,"rankEligible":true,"score":140.43,"scoreLo":137.5,"scoreHi":143.45,"modelScoreStateId":"north-mini-code-1-0::default","modelScoreConfigSlug":"north-mini-code-1-0","modelScoreStateName":"North Mini Code","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":234.9722,"rankLo":207.0,"rankHi":271.0},{"slug":"deepseek-r1-0528","modelId":"deepseek-r1-0528","modelSlug":"deepseek-r1-0528","name":"DeepSeek-R1-0528","organizationSlug":"deepseek","n":76,"rankEligible":true,"score":140.39,"scoreLo":138.69,"scoreHi":142.23,"modelScoreStateId":"deepseek-r1-0528::default","modelScoreConfigSlug":"deepseek-r1-0528","modelScoreStateName":"DeepSeek-R1-0528","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":234.6056,"rankLo":220.0,"rankHi":251.0},{"slug":"ling-2-6-1t","modelId":"ling-2-6-1t","modelSlug":"ling-2-6-1t","name":"Ling 2.6 1T","organizationSlug":"inclusion-ai","n":10,"rankEligible":true,"score":140.39,"scoreLo":135.83,"scoreHi":144.56,"modelScoreStateId":"ling-2-6-1t::default","modelScoreConfigSlug":"ling-2-6-1t","modelScoreStateName":"Ling 2.6 1T","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":236.4431,"rankLo":194.0,"rankHi":292.0},{"slug":"glm-4-5-air","modelId":"glm-4-5-air","modelSlug":"glm-4-5-air","name":"GLM 4.5 Air","organizationSlug":"zhipu-ai","n":23,"rankEligible":true,"score":140.32,"scoreLo":137.84,"scoreHi":142.96,"modelScoreStateId":"glm-4-5-air::default","modelScoreConfigSlug":"glm-4-5-air","modelScoreStateName":"GLM 4.5 Air","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":235.9606,"rankLo":211.0,"rankHi":266.0},{"slug":"nemotron-cascade-2-30b-a3b","modelId":"nemotron-cascade-2-30b-a3b","modelSlug":"nemotron-cascade-2-30b-a3b","name":"Nemotron Cascade 2 30B A3B","organizationSlug":"nvidia","n":21,"rankEligible":true,"score":140.12,"scoreLo":137.8,"scoreHi":142.63,"modelScoreStateId":"nemotron-cascade-2-30b-a3b::default","modelScoreConfigSlug":"nemotron-cascade-2-30b-a3b","modelScoreStateName":"Nemotron Cascade 2 30B A3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":238.4533,"rankLo":215.0,"rankHi":265.0},{"slug":"trinity-large-thinking","modelId":"trinity-large-thinking","modelSlug":"trinity-large-thinking","name":"Trinity Large Thinking","organizationSlug":"arcee-ai","n":27,"rankEligible":true,"score":140.07,"scoreLo":137.77,"scoreHi":142.44,"modelScoreStateId":"trinity-large-thinking::default","modelScoreConfigSlug":"trinity-large-thinking","modelScoreStateName":"Trinity Large Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":238.9444,"rankLo":216.0,"rankHi":264.0},{"slug":"qwen3-vl-32b-thinking","modelId":"qwen3-vl-32b-thinking","modelSlug":"qwen3-vl-32b-thinking","name":"Qwen3 VL 32B Thinking","organizationSlug":"alibaba","n":50,"rankEligible":true,"score":140.04,"scoreLo":137.92,"scoreHi":142.25,"modelScoreStateId":"qwen3-vl-32b-thinking::default","modelScoreConfigSlug":"qwen3-vl-32b-thinking","modelScoreStateName":"Qwen3 VL 32B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":239.4122,"rankLo":218.0,"rankHi":262.0},{"slug":"internvl3-5-241b-a28b","modelId":"internvl3-5-241b-a28b","modelSlug":"internvl3-5-241b-a28b","name":"InternVL3.5-241B-A28B","organizationSlug":"opengvlab","n":11,"rankEligible":true,"score":139.99,"scoreLo":135.97,"scoreHi":144.26,"modelScoreStateId":"internvl3-5-241b-a28b::default","modelScoreConfigSlug":"internvl3-5-241b-a28b","modelScoreStateName":"InternVL3.5-241B-A28B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":241.3683,"rankLo":197.0,"rankHi":292.0},{"slug":"granite-4-2-30b","modelId":"granite-4-2-30b","modelSlug":"granite-4-2-30b","name":"Granite 4.2 30B","organizationSlug":"ibm","n":11,"rankEligible":true,"score":139.81,"scoreLo":136.42,"scoreHi":143.03,"modelScoreStateId":"granite-4-2-30b::enabled","modelScoreConfigSlug":"granite-4-2-30b-thinking","modelScoreStateName":"Granite 4.2 30B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":243.0453,"rankLo":209.0,"rankHi":284.025},{"slug":"qwen3-235b-a22b","modelId":"qwen3-235b-a22b","modelSlug":"qwen3-235b-a22b","name":"Qwen3-235B-A22B","organizationSlug":"alibaba","n":36,"rankEligible":true,"score":139.73,"scoreLo":138.04,"scoreHi":141.48,"modelScoreStateId":"qwen3-235b-a22b::enabled","modelScoreConfigSlug":"qwen3-235b-a22b-thinking","modelScoreStateName":"Qwen3-235B-A22B (thinking)","modelScoreStateBestProbability":0.9425,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":243.4011,"rankLo":228.0,"rankHi":259.0},{"slug":"kimi-k2-instruct","modelId":"kimi-k2-instruct","modelSlug":"kimi-k2-instruct","name":"Kimi K2 Instruct","organizationSlug":"moonshot-ai","n":73,"rankEligible":true,"score":139.72,"scoreLo":137.89,"scoreHi":141.65,"modelScoreStateId":"kimi-k2-instruct::default","modelScoreConfigSlug":"kimi-k2-instruct","modelScoreStateName":"Kimi K2 Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":243.6072,"rankLo":225.0,"rankHi":262.0},{"slug":"mercury-2","modelId":"mercury-2","modelSlug":"mercury-2","name":"Mercury 2","organizationSlug":"inception-labs","n":19,"rankEligible":true,"score":139.68,"scoreLo":136.91,"scoreHi":142.55,"modelScoreStateId":"mercury-2::default","modelScoreConfigSlug":"mercury-2","modelScoreStateName":"Mercury 2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":244.4375,"rankLo":215.0,"rankHi":278.0},{"slug":"devstral-2-123b-instruct-2512","modelId":"devstral-2-123b-instruct-2512","modelSlug":"devstral-2-123b-instruct-2512","name":"Devstral 2","organizationSlug":"mistral-ai","n":29,"rankEligible":true,"score":139.47,"scoreLo":136.99,"scoreHi":141.98,"modelScoreStateId":"devstral-2-123b-instruct-2512::default","modelScoreConfigSlug":"devstral-2-123b-instruct-2512","modelScoreStateName":"Devstral 2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":246.9981,"rankLo":220.0,"rankHi":277.0},{"slug":"nemotron-3-5-lightning-30b-a3b-nvfp4","modelId":"nemotron-3-5-lightning-30b-a3b-nvfp4","modelSlug":"nemotron-3-5-lightning-30b-a3b-nvfp4","name":"Nemotron 3.5 Lightning 30B A3B NVFP4","organizationSlug":"nvidia","n":11,"rankEligible":true,"score":139.4,"scoreLo":136.31,"scoreHi":142.56,"modelScoreStateId":"nemotron-3-5-lightning-30b-a3b-nvfp4::default","modelScoreConfigSlug":"nemotron-3-5-lightning-30b-a3b-nvfp4","modelScoreStateName":"Nemotron 3.5 Lightning 30B A3B NVFP4","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":248.2764,"rankLo":214.0,"rankHi":287.0},{"slug":"gpt-5-1-chat-latest","modelId":"gpt-5-1-chat-latest","modelSlug":"gpt-5-1-chat-latest","name":"GPT-5.1 Instant","organizationSlug":"openai","n":10,"rankEligible":true,"score":139.3,"scoreLo":135.62,"scoreHi":142.91,"modelScoreStateId":"gpt-5-1-chat-latest::default","modelScoreConfigSlug":"gpt-5-1-chat-latest","modelScoreStateName":"GPT-5.1 Instant","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":249.8097,"rankLo":210.0,"rankHi":298.0},{"slug":"lfm2-5-24b-a2b","modelId":"lfm2-5-24b-a2b","modelSlug":"lfm2-5-24b-a2b","name":"LFM2.5-24B-A2B","organizationSlug":"liquid-ai","n":8,"rankEligible":true,"score":139.26,"scoreLo":135.68,"scoreHi":142.89,"modelScoreStateId":"lfm2-5-24b-a2b::default","modelScoreConfigSlug":"lfm2-5-24b-a2b","modelScoreStateName":"LFM2.5-24B-A2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":250.4314,"rankLo":211.0,"rankHi":296.0},{"slug":"qwen3-next-80b-a3b-thinking","modelId":"qwen3-next-80b-a3b-thinking","modelSlug":"qwen3-next-80b-a3b-thinking","name":"Qwen3-Next-80B-A3B-Thinking","organizationSlug":"alibaba","n":57,"rankEligible":true,"score":139.21,"scoreLo":137.32,"scoreHi":141.2,"modelScoreStateId":"qwen3-next-80b-a3b-thinking::default","modelScoreConfigSlug":"qwen3-next-80b-a3b-thinking","modelScoreStateName":"Qwen3-Next-80B-A3B-Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":250.5864,"rankLo":231.0,"rankHi":270.0},{"slug":"laguna-m-1","modelId":"laguna-m-1","modelSlug":"laguna-m-1","name":"Laguna M.1","organizationSlug":"poolside","n":20,"rankEligible":true,"score":139.13,"scoreLo":136.4,"scoreHi":141.83,"modelScoreStateId":"laguna-m-1::default","modelScoreConfigSlug":"laguna-m-1","modelScoreStateName":"Laguna M.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":251.7883,"rankLo":220.0,"rankHi":285.025},{"slug":"nova-2-lite-v1-0","modelId":"nova-2-lite-v1-0","modelSlug":"nova-2-lite-v1-0","name":"Nova 2 Lite","organizationSlug":"amazon","n":12,"rankEligible":true,"score":139.03,"scoreLo":136.48,"scoreHi":141.75,"modelScoreStateId":"nova-2-lite-v1-0::effort:high","modelScoreConfigSlug":"nova-2-lite-v1-0-high","modelScoreStateName":"Nova 2 Lite (high)","modelScoreStateBestProbability":0.6247222222222222,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":253.0711,"rankLo":223.0,"rankHi":284.0},{"slug":"seed-oss-36b-instruct","modelId":"seed-oss-36b-instruct","modelSlug":"seed-oss-36b-instruct","name":"Seed OSS 36B Instruct","organizationSlug":"bytedance","n":11,"rankEligible":true,"score":138.95,"scoreLo":135.69,"scoreHi":142.16,"modelScoreStateId":"seed-oss-36b-instruct::default","modelScoreConfigSlug":"seed-oss-36b-instruct","modelScoreStateName":"Seed OSS 36B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":254.3169,"rankLo":216.0,"rankHi":295.0},{"slug":"ernie-5-0","modelId":"ernie-5-0","modelSlug":"ernie-5-0","name":"ERNIE 5.0","organizationSlug":"baidu","n":5,"rankEligible":true,"score":138.87,"scoreLo":134.4,"scoreHi":142.96,"modelScoreStateId":"ernie-5-0::default","modelScoreConfigSlug":"ernie-5-0","modelScoreStateName":"ERNIE 5.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":255.7675,"rankLo":210.0,"rankHi":313.0},{"slug":"o3-mini","modelId":"o3-mini","modelSlug":"o3-mini","name":"o3-mini","organizationSlug":"openai","n":77,"rankEligible":true,"score":138.8,"scoreLo":137.07,"scoreHi":140.62,"modelScoreStateId":"o3-mini::effort:high","modelScoreConfigSlug":"o3-mini-high","modelScoreStateName":"o3-mini (high)","modelScoreStateBestProbability":0.86,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":256.1206,"rankLo":240.0,"rankHi":273.0},{"slug":"kimi-k2-instruct-0905","modelId":"kimi-k2-instruct-0905","modelSlug":"kimi-k2-instruct-0905","name":"Kimi K2 Instruct 0905","organizationSlug":"moonshot-ai","n":42,"rankEligible":true,"score":138.78,"scoreLo":136.64,"scoreHi":141.08,"modelScoreStateId":"kimi-k2-instruct-0905::default","modelScoreConfigSlug":"kimi-k2-instruct-0905","modelScoreStateName":"Kimi K2 Instruct 0905","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":256.4411,"rankLo":232.0,"rankHi":281.0},{"slug":"hypernova-60b-2605","modelId":"hypernova-60b-2605","modelSlug":"hypernova-60b-2605","name":"HyperNova 60B 2605","organizationSlug":"multiverse-computing","n":10,"rankEligible":true,"score":138.76,"scoreLo":135.04,"scoreHi":142.35,"modelScoreStateId":"hypernova-60b-2605::effort:high","modelScoreConfigSlug":"hypernova-60b-2605","modelScoreStateName":"HyperNova 60B 2605 (high)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":256.8983,"rankLo":216.0,"rankHi":305.025},{"slug":"qwen3-next-80b-a3b-instruct","modelId":"qwen3-next-80b-a3b-instruct","modelSlug":"qwen3-next-80b-a3b-instruct","name":"Qwen3 Next 80B A3B Instruct","organizationSlug":"alibaba","n":34,"rankEligible":true,"score":138.7,"scoreLo":136.54,"scoreHi":140.83,"modelScoreStateId":"qwen3-next-80b-a3b-instruct::default","modelScoreConfigSlug":"qwen3-next-80b-a3b-instruct","modelScoreStateName":"Qwen3 Next 80B A3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":257.6225,"rankLo":234.0,"rankHi":282.0},{"slug":"qwen3-5-omni-flash","modelId":"qwen3-5-omni-flash","modelSlug":"qwen3-5-omni-flash","name":"Qwen3.5-Omni-Flash","organizationSlug":"alibaba","n":41,"rankEligible":true,"score":138.64,"scoreLo":136.52,"scoreHi":140.78,"modelScoreStateId":"qwen3-5-omni-flash::default","modelScoreConfigSlug":"qwen3-5-omni-flash","modelScoreStateName":"Qwen3.5-Omni-Flash","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":258.3786,"rankLo":234.975,"rankHi":282.0},{"slug":"grok-code-fast-1","modelId":"grok-code-fast-1","modelSlug":"grok-code-fast-1","name":"Grok Code Fast 1","organizationSlug":"xai","n":23,"rankEligible":true,"score":138.6,"scoreLo":135.91,"scoreHi":141.23,"modelScoreStateId":"grok-code-fast-1::default","modelScoreConfigSlug":"grok-code-fast-1","modelScoreStateName":"Grok Code Fast 1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":258.8942,"rankLo":228.0,"rankHi":292.0},{"slug":"qwen3-vl-32b-instruct","modelId":"qwen3-vl-32b-instruct","modelSlug":"qwen3-vl-32b-instruct","name":"Qwen3 VL 32B Instruct","organizationSlug":"alibaba","n":50,"rankEligible":true,"score":138.46,"scoreLo":136.49,"scoreHi":140.51,"modelScoreStateId":"qwen3-vl-32b-instruct::default","modelScoreConfigSlug":"qwen3-vl-32b-instruct","modelScoreStateName":"Qwen3 VL 32B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":260.715,"rankLo":239.0,"rankHi":283.025},{"slug":"ling-3-0-tiny","modelId":"ling-3-0-tiny","modelSlug":"ling-3-0-tiny","name":"Ling 3.0 Tiny","organizationSlug":"inclusion-ai","n":10,"rankEligible":true,"score":138.41,"scoreLo":134.67,"scoreHi":141.95,"modelScoreStateId":"ling-3-0-tiny::enabled","modelScoreConfigSlug":"ling-3-0-tiny-thinking","modelScoreStateName":"Ling 3.0 Tiny (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":261.7239,"rankLo":219.0,"rankHi":309.0},{"slug":"glm-4-7-flash","modelId":"glm-4-7-flash","modelSlug":"glm-4-7-flash","name":"GLM 4.7 Flash","organizationSlug":"zhipu-ai","n":5,"rankEligible":true,"score":138.35,"scoreLo":134.8,"scoreHi":141.94,"modelScoreStateId":"glm-4-7-flash::enabled","modelScoreConfigSlug":"glm-4-7-flash-thinking","modelScoreStateName":"GLM 4.7 Flash (thinking)","modelScoreStateBestProbability":0.9494444444444444,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":262.5311,"rankLo":220.0,"rankHi":308.0},{"slug":"gpt-4-5-preview","modelId":"gpt-4-5-preview","modelSlug":"gpt-4-5-preview","name":"GPT-4.5","organizationSlug":"openai","n":51,"rankEligible":true,"score":138.28,"scoreLo":136.24,"scoreHi":140.32,"modelScoreStateId":"gpt-4-5-preview::default","modelScoreConfigSlug":"gpt-4-5-preview","modelScoreStateName":"GPT-4.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":263.3442,"rankLo":242.0,"rankHi":286.0},{"slug":"diffusiongemma-26b-a4b-it","modelId":"diffusiongemma-26b-a4b-it","modelSlug":"diffusiongemma-26b-a4b-it","name":"DiffusionGemma 26B A4B IT","organizationSlug":"google","n":24,"rankEligible":true,"score":138.19,"scoreLo":136.16,"scoreHi":140.19,"modelScoreStateId":"diffusiongemma-26b-a4b-it::enabled","modelScoreConfigSlug":"diffusiongemma-26b-a4b-it-thinking","modelScoreStateName":"DiffusionGemma 26B A4B IT (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":264.6442,"rankLo":244.0,"rankHi":286.0},{"slug":"command-a-plus-05-2026-bf16","modelId":"command-a-plus-05-2026-bf16","modelSlug":"command-a-plus-05-2026-bf16","name":"Command A+ (May 2026, BF16)","organizationSlug":"cohere","n":22,"rankEligible":true,"score":138.16,"scoreLo":135.34,"scoreHi":141.08,"modelScoreStateId":"command-a-plus-05-2026-bf16::default","modelScoreConfigSlug":"command-a-plus-05-2026-bf16","modelScoreStateName":"Command A+ (May 2026, BF16)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":264.9372,"rankLo":231.0,"rankHi":301.0},{"slug":"intellect-3","modelId":"intellect-3","modelSlug":"intellect-3","name":"INTELLECT-3","organizationSlug":"prime-intellect","n":13,"rankEligible":true,"score":137.97,"scoreLo":134.46,"scoreHi":141.47,"modelScoreStateId":"intellect-3::default","modelScoreConfigSlug":"intellect-3","modelScoreStateName":"INTELLECT-3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":267.5264,"rankLo":223.0,"rankHi":313.0},{"slug":"o1-preview","modelId":"o1-preview","modelSlug":"o1-preview","name":"o1 Preview","organizationSlug":"openai","n":32,"rankEligible":true,"score":137.93,"scoreLo":135.18,"scoreHi":140.67,"modelScoreStateId":"o1-preview::default","modelScoreConfigSlug":"o1-preview","modelScoreStateName":"o1 Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":268.1853,"rankLo":235.0,"rankHi":301.0},{"slug":"nemotron-3-5-lightning-30b-a3b-bf16","modelId":"nemotron-3-5-lightning-30b-a3b-bf16","modelSlug":"nemotron-3-5-lightning-30b-a3b-bf16","name":"Nemotron 3.5 Lightning 30B A3B BF16","organizationSlug":"nvidia","n":20,"rankEligible":true,"score":137.91,"scoreLo":135.32,"scoreHi":140.45,"modelScoreStateId":"nemotron-3-5-lightning-30b-a3b-bf16::default","modelScoreConfigSlug":"nemotron-3-5-lightning-30b-a3b-bf16","modelScoreStateName":"Nemotron 3.5 Lightning 30B A3B BF16","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":268.3986,"rankLo":239.0,"rankHi":299.0},{"slug":"gpt-5-chat-latest","modelId":"gpt-5-chat-latest","modelSlug":"gpt-5-chat-latest","name":"GPT-5 Chat","organizationSlug":"openai","n":27,"rankEligible":true,"score":137.9,"scoreLo":135.39,"scoreHi":140.53,"modelScoreStateId":"gpt-5-chat-latest::default","modelScoreConfigSlug":"gpt-5-chat-latest","modelScoreStateName":"GPT-5 Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":268.5417,"rankLo":238.0,"rankHi":298.0},{"slug":"gpt-4-1","modelId":"gpt-4-1","modelSlug":"gpt-4-1","name":"GPT-4.1","organizationSlug":"openai","n":131,"rankEligible":true,"score":137.58,"scoreLo":135.97,"scoreHi":139.18,"modelScoreStateId":"gpt-4-1::default","modelScoreConfigSlug":"gpt-4-1","modelScoreStateName":"GPT-4.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":272.9767,"rankLo":258.0,"rankHi":288.0},{"slug":"ling-1t","modelId":"ling-1t","modelSlug":"ling-1t","name":"Ling-1T","organizationSlug":"inclusion-ai","n":10,"rankEligible":true,"score":137.54,"scoreLo":133.45,"scoreHi":141.57,"modelScoreStateId":"ling-1t::default","modelScoreConfigSlug":"ling-1t","modelScoreStateName":"Ling-1T","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":273.5458,"rankLo":223.0,"rankHi":329.0},{"slug":"cogito-671b-v2-1","modelId":"cogito-671b-v2-1","modelSlug":"cogito-671b-v2-1","name":"Cogito v2.1 671B","organizationSlug":"deepcogito","n":8,"rankEligible":true,"score":137.53,"scoreLo":133.04,"scoreHi":142.03,"modelScoreStateId":"cogito-671b-v2-1::enabled","modelScoreConfigSlug":"cogito-671b-v2-1-thinking","modelScoreStateName":"Cogito v2.1 671B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":273.7861,"rankLo":217.0,"rankHi":334.025},{"slug":"gemini-2-5-flash-lite-preview-09-2025","modelId":"gemini-2-5-flash-lite-preview-09-2025","modelSlug":"gemini-2-5-flash-lite-preview-09-2025","name":"Gemini 2.5 Flash-Lite Preview (09-2025)","organizationSlug":"google","n":28,"rankEligible":true,"score":137.52,"scoreLo":135.36,"scoreHi":139.78,"modelScoreStateId":"gemini-2-5-flash-lite-preview-09-2025::enabled","modelScoreConfigSlug":"gemini-2-5-flash-lite-2025-09-thinking","modelScoreStateName":"Gemini 2.5 Flash-Lite Preview (09-2025) (thinking)","modelScoreStateBestProbability":0.9627777777777777,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":273.6408,"rankLo":249.0,"rankHi":298.0},{"slug":"gemini-2-5-flash-preview-04-17","modelId":"gemini-2-5-flash-preview-04-17","modelSlug":"gemini-2-5-flash-preview-04-17","name":"Gemini 2.5 Flash Preview 04-17","organizationSlug":"google","n":20,"rankEligible":true,"score":137.47,"scoreLo":135.13,"scoreHi":139.78,"modelScoreStateId":"gemini-2-5-flash-preview-04-17::enabled","modelScoreConfigSlug":"gemini-2-5-flash-preview-04-17-thinking","modelScoreStateName":"Gemini 2.5 Flash Preview 04-17 (thinking)","modelScoreStateBestProbability":0.9755555555555555,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":274.4103,"rankLo":248.0,"rankHi":300.0},{"slug":"nemotron-3-nano-30b-a3b-bf16","modelId":"nemotron-3-nano-30b-a3b-bf16","modelSlug":"nemotron-3-nano-30b-a3b-bf16","name":"Nemotron 3 Nano 30B A3B","organizationSlug":"nvidia","n":15,"rankEligible":true,"score":137.47,"scoreLo":135.01,"scoreHi":140.01,"modelScoreStateId":"nemotron-3-nano-30b-a3b-bf16::enabled","modelScoreConfigSlug":"nemotron-3-nano-30b-a3b-bf16-thinking","modelScoreStateName":"Nemotron 3 Nano 30B A3B (thinking)","modelScoreStateBestProbability":0.8222222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":274.42,"rankLo":246.0,"rankHi":303.0},{"slug":"grok-3","modelId":"grok-3","modelSlug":"grok-3","name":"Grok 3 (API)","organizationSlug":"xai","n":30,"rankEligible":true,"score":137.18,"scoreLo":134.76,"scoreHi":139.59,"modelScoreStateId":"grok-3::default","modelScoreConfigSlug":"grok-3","modelScoreStateName":"Grok 3 (API)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"none","modelScoreHarness":"none","rankMean":278.2706,"rankLo":252.0,"rankHi":306.0},{"slug":"nova-2-omni-v1-0","modelId":"nova-2-omni-v1-0","modelSlug":"nova-2-omni-v1-0","name":"Nova 2 Omni (Preview)","organizationSlug":"amazon","n":9,"rankEligible":true,"score":137.18,"scoreLo":134.3,"scoreHi":140.13,"modelScoreStateId":"nova-2-omni-v1-0::effort:medium","modelScoreConfigSlug":"nova-2-omni-v1-0-medium","modelScoreStateName":"Nova 2 Omni (Preview) (medium)","modelScoreStateBestProbability":0.7983333333333333,"modelScoreReasoningCondition":"effort:medium","modelScoreReasoningEffort":"medium","modelScoreHarness":"none","rankMean":278.3042,"rankLo":242.0,"rankHi":312.0},{"slug":"deepseek-r1","modelId":"deepseek-r1","modelSlug":"deepseek-r1","name":"DeepSeek-R1","organizationSlug":"deepseek","n":90,"rankEligible":true,"score":137.13,"scoreLo":135.46,"scoreHi":138.85,"modelScoreStateId":"deepseek-r1::default","modelScoreConfigSlug":"deepseek-r1","modelScoreStateName":"DeepSeek-R1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":279.1297,"rankLo":263.0,"rankHi":294.0},{"slug":"qwen3-vl-30b-a3b-thinking","modelId":"qwen3-vl-30b-a3b-thinking","modelSlug":"qwen3-vl-30b-a3b-thinking","name":"Qwen3 VL 30B A3B Thinking","organizationSlug":"alibaba","n":57,"rankEligible":true,"score":136.99,"scoreLo":134.93,"scoreHi":139.07,"modelScoreStateId":"qwen3-vl-30b-a3b-thinking::default","modelScoreConfigSlug":"qwen3-vl-30b-a3b-thinking","modelScoreStateName":"Qwen3 VL 30B A3B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":280.9356,"rankLo":258.0,"rankHi":302.0},{"slug":"minicpm5-2b","modelId":"minicpm5-2b","modelSlug":"minicpm5-2b","name":"MiniCPM5-2B","organizationSlug":"openbmb","n":9,"rankEligible":true,"score":136.96,"scoreLo":133.17,"scoreHi":140.59,"modelScoreStateId":"minicpm5-2b::enabled","modelScoreConfigSlug":"minicpm5-2b-thinking","modelScoreStateName":"MiniCPM5-2B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":281.3669,"rankLo":234.0,"rankHi":332.0},{"slug":"gemini-exp-1206","modelId":"gemini-exp-1206","modelSlug":"gemini-exp-1206","name":"Gemini Exp-1206","organizationSlug":"google","n":8,"rankEligible":true,"score":136.9,"scoreLo":132.3,"scoreHi":141.42,"modelScoreStateId":"gemini-exp-1206::default","modelScoreConfigSlug":"gemini-exp-1206","modelScoreStateName":"Gemini Exp-1206","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":282.2356,"rankLo":224.0,"rankHi":341.025},{"slug":"qwen3-30b-a3b-thinking-2507","modelId":"qwen3-30b-a3b-thinking-2507","modelSlug":"qwen3-30b-a3b-thinking-2507","name":"Qwen3 30B A3B Thinking 2507","organizationSlug":"alibaba","n":62,"rankEligible":true,"score":136.86,"scoreLo":135.09,"scoreHi":138.61,"modelScoreStateId":"qwen3-30b-a3b-thinking-2507::default","modelScoreConfigSlug":"qwen3-30b-a3b-2507-thinking","modelScoreStateName":"Qwen3 30B A3B Thinking 2507","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":282.7572,"rankLo":265.0,"rankHi":301.0},{"slug":"lfm3-1-7b","modelId":"lfm3-1-7b","modelSlug":"lfm3-1-7b","name":"LFM3-1.7B","organizationSlug":"liquid-ai","n":13,"rankEligible":true,"score":136.84,"scoreLo":133.45,"scoreHi":140.11,"modelScoreStateId":"lfm3-1-7b::default","modelScoreConfigSlug":"lfm3-1-7b","modelScoreStateName":"LFM3-1.7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":283.0044,"rankLo":243.0,"rankHi":326.0},{"slug":"ring-1t","modelId":"ring-1t","modelSlug":"ring-1t","name":"Ring-1T","organizationSlug":"inclusion-ai","n":9,"rankEligible":true,"score":136.69,"scoreLo":132.54,"scoreHi":140.66,"modelScoreStateId":"ring-1t::default","modelScoreConfigSlug":"ring-1t","modelScoreStateName":"Ring-1T","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":285.0467,"rankLo":234.0,"rankHi":339.0},{"slug":"gpt-5-nano","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano","organizationSlug":"openai","n":19,"rankEligible":true,"score":136.51,"scoreLo":134.55,"scoreHi":138.36,"modelScoreStateId":"gpt-5-nano::effort:medium","modelScoreConfigSlug":"gpt-5-nano-medium","modelScoreStateName":"GPT-5 Nano (medium)","modelScoreStateBestProbability":0.8002777777777778,"modelScoreReasoningCondition":"effort:medium","modelScoreReasoningEffort":"medium","modelScoreHarness":"none","rankMean":287.4592,"rankLo":268.0,"rankHi":307.0},{"slug":"hermes-4-405b","modelId":"hermes-4-405b","modelSlug":"hermes-4-405b","name":"Hermes 4 405B","organizationSlug":"nous-research","n":7,"rankEligible":true,"score":136.41,"scoreLo":133.03,"scoreHi":139.83,"modelScoreStateId":"hermes-4-405b::enabled","modelScoreConfigSlug":"hermes-4-llama-3-1-405b-thinking","modelScoreStateName":"Hermes 4 405B (thinking)","modelScoreStateBestProbability":0.69,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":288.8006,"rankLo":246.0,"rankHi":333.0},{"slug":"mistral-small-4-119b-2603","modelId":"mistral-small-4-119b-2603","modelSlug":"mistral-small-4-119b-2603","name":"Mistral Small 4","organizationSlug":"mistral-ai","n":28,"rankEligible":true,"score":136.21,"scoreLo":134.03,"scoreHi":138.44,"modelScoreStateId":"mistral-small-4-119b-2603::effort:high","modelScoreConfigSlug":"mistral-small-4-119b-2603-high","modelScoreStateName":"Mistral Small 4 (high)","modelScoreStateBestProbability":0.9897222222222222,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":291.5194,"rankLo":266.0,"rankHi":317.0},{"slug":"qed-nano","modelId":"qed-nano","modelSlug":"qed-nano","name":"QED-Nano","organizationSlug":"lm-provers","n":9,"rankEligible":true,"score":136.14,"scoreLo":133.4,"scoreHi":138.94,"modelScoreStateId":"qed-nano::default","modelScoreConfigSlug":"qed-nano-matharena","modelScoreStateName":"QED-Nano","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":292.3933,"rankLo":258.0,"rankHi":326.0},{"slug":"gemini-exp-1121","modelId":"gemini-exp-1121","modelSlug":"gemini-exp-1121","name":"Gemini Experimental 1121","organizationSlug":"google","n":5,"rankEligible":true,"score":136.08,"scoreLo":127.8,"scoreHi":144.98,"modelScoreStateId":"gemini-exp-1121::default","modelScoreConfigSlug":"gemini-exp-1121","modelScoreStateName":"Gemini Experimental 1121","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":294.2008,"rankLo":184.0,"rankHi":404.0},{"slug":"ministral-3-14b-reasoning-2512","modelId":"ministral-3-14b-reasoning-2512","modelSlug":"ministral-3-14b-reasoning-2512","name":"Ministral 3 14B Reasoning 2512","organizationSlug":"mistral-ai","n":6,"rankEligible":true,"score":136.07,"scoreLo":131.48,"scoreHi":140.76,"modelScoreStateId":"ministral-3-14b-reasoning-2512::default","modelScoreConfigSlug":"ministral-3-14b-reasoning-2512","modelScoreStateName":"Ministral 3 14B Reasoning 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":293.4522,"rankLo":233.0,"rankHi":350.025},{"slug":"laguna-xs-2","modelId":"laguna-xs-2","modelSlug":"laguna-xs-2","name":"Laguna XS.2","organizationSlug":"poolside","n":19,"rankEligible":true,"score":136.05,"scoreLo":133.28,"scoreHi":138.8,"modelScoreStateId":"laguna-xs-2::default","modelScoreConfigSlug":"laguna-xs-2","modelScoreStateName":"Laguna XS.2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":293.6836,"rankLo":259.0,"rankHi":329.0},{"slug":"step3-vl-10b","modelId":"step3-vl-10b","modelSlug":"step3-vl-10b","name":"Step3-VL-10B","organizationSlug":"stepfun","n":4,"rankEligible":true,"score":136.01,"scoreLo":131.16,"scoreHi":140.49,"modelScoreStateId":"step3-vl-10b::default","modelScoreConfigSlug":"step3-vl-10b","modelScoreStateName":"Step3-VL-10B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":294.32,"rankLo":236.975,"rankHi":355.0},{"slug":"nemotron-cascade-8b","modelId":"nemotron-cascade-8b","modelSlug":"nemotron-cascade-8b","name":"Nemotron Cascade 8B","organizationSlug":"nvidia","n":8,"rankEligible":true,"score":135.82,"scoreLo":131.86,"scoreHi":139.76,"modelScoreStateId":"nemotron-cascade-8b::default","modelScoreConfigSlug":"nemotron-cascade-8b","modelScoreStateName":"Nemotron Cascade 8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":297.0039,"rankLo":245.0,"rankHi":347.0},{"slug":"magistral-medium-2509","modelId":"magistral-medium-2509","modelSlug":"magistral-medium-2509","name":"Magistral Medium 1.2","organizationSlug":"mistral-ai","n":27,"rankEligible":true,"score":135.66,"scoreLo":133.12,"scoreHi":138.23,"modelScoreStateId":"magistral-medium-2509::default","modelScoreConfigSlug":"magistral-medium-2509","modelScoreStateName":"Magistral Medium 1.2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":299.0139,"rankLo":269.0,"rankHi":330.0},{"slug":"glm-4-6v","modelId":"glm-4-6v","modelSlug":"glm-4-6v","name":"GLM-4.6V","organizationSlug":"zhipu-ai","n":12,"rankEligible":true,"score":135.6,"scoreLo":132.26,"scoreHi":138.99,"modelScoreStateId":"glm-4-6v::default","modelScoreConfigSlug":"glm-4-6v","modelScoreStateName":"GLM-4.6V","modelScoreStateBestProbability":0.96,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":299.9731,"rankLo":258.0,"rankHi":343.0},{"slug":"minimax-m1-80k","modelId":"minimax-m1-80k","modelSlug":"minimax-m1-80k","name":"MiniMax M1 80k","organizationSlug":"minimax","n":7,"rankEligible":true,"score":135.58,"scoreLo":131.04,"scoreHi":139.86,"modelScoreStateId":"minimax-m1-80k::effort:80k","modelScoreConfigSlug":"minimax-m1-80k","modelScoreStateName":"MiniMax M1 80k (80k)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:80k","modelScoreReasoningEffort":"80k","modelScoreHarness":"none","rankMean":300.1161,"rankLo":245.0,"rankHi":356.0},{"slug":"devstral-small-2-24b-instruct-2512","modelId":"devstral-small-2-24b-instruct-2512","modelSlug":"devstral-small-2-24b-instruct-2512","name":"Devstral Small 2","organizationSlug":"mistral-ai","n":22,"rankEligible":true,"score":135.49,"scoreLo":132.56,"scoreHi":138.23,"modelScoreStateId":"devstral-small-2-24b-instruct-2512::default","modelScoreConfigSlug":"devstral-small-2-24b-instruct-2512","modelScoreStateName":"Devstral Small 2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":301.4217,"rankLo":268.0,"rankHi":338.0},{"slug":"glm-4-5v","modelId":"glm-4-5v","modelSlug":"glm-4-5v","name":"GLM-4.5V","organizationSlug":"zhipu-ai","n":17,"rankEligible":true,"score":135.39,"scoreLo":132.63,"scoreHi":138.07,"modelScoreStateId":"glm-4-5v::enabled","modelScoreConfigSlug":"glm-4-5v-thinking","modelScoreStateName":"GLM-4.5V (thinking)","modelScoreStateBestProbability":0.9755555555555555,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":302.8233,"rankLo":271.0,"rankHi":337.0},{"slug":"falcon-h1r-7b","modelId":"falcon-h1r-7b","modelSlug":"falcon-h1r-7b","name":"Falcon H1R 7B","organizationSlug":"tii","n":12,"rankEligible":true,"score":135.35,"scoreLo":132.41,"scoreHi":138.13,"modelScoreStateId":"falcon-h1r-7b::default","modelScoreConfigSlug":"falcon-h1r-7b","modelScoreStateName":"Falcon H1R 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":303.3283,"rankLo":270.0,"rankHi":339.0},{"slug":"granite-4-2-8b","modelId":"granite-4-2-8b","modelSlug":"granite-4-2-8b","name":"Granite 4.2 8B","organizationSlug":"ibm","n":11,"rankEligible":true,"score":135.26,"scoreLo":131.84,"scoreHi":138.72,"modelScoreStateId":"granite-4-2-8b::enabled","modelScoreConfigSlug":"granite-4-2-8b-thinking","modelScoreStateName":"Granite 4.2 8B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":304.5317,"rankLo":262.0,"rankHi":348.0},{"slug":"llama-3-3-nemotron-super-49b-v1-5","modelId":"llama-3-3-nemotron-super-49b-v1-5","modelSlug":"llama-3-3-nemotron-super-49b-v1-5","name":"Llama 3.3 Nemotron Super 49B V1.5","organizationSlug":"nvidia","n":13,"rankEligible":true,"score":135.15,"scoreLo":131.88,"scoreHi":138.32,"modelScoreStateId":"llama-3-3-nemotron-super-49b-v1-5::enabled","modelScoreConfigSlug":"llama-3-3-nemotron-super-49b-v1-5-thinking","modelScoreStateName":"Llama 3.3 Nemotron Super 49B V1.5 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":306.0681,"rankLo":266.0,"rankHi":347.0},{"slug":"qwen3-vl-30b-a3b-instruct","modelId":"qwen3-vl-30b-a3b-instruct","modelSlug":"qwen3-vl-30b-a3b-instruct","name":"Qwen3 VL 30B A3B Instruct","organizationSlug":"alibaba","n":50,"rankEligible":true,"score":135.1,"scoreLo":133.04,"scoreHi":137.11,"modelScoreStateId":"qwen3-vl-30b-a3b-instruct::default","modelScoreConfigSlug":"qwen3-vl-30b-a3b-instruct","modelScoreStateName":"Qwen3 VL 30B A3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":306.6497,"rankLo":285.0,"rankHi":331.0},{"slug":"nemotron-cascade-14b-thinking","modelId":"nemotron-cascade-14b-thinking","modelSlug":"nemotron-cascade-14b-thinking","name":"Nemotron Cascade 14B Thinking","organizationSlug":"nvidia","n":9,"rankEligible":true,"score":135.09,"scoreLo":131.36,"scoreHi":138.92,"modelScoreStateId":"nemotron-cascade-14b-thinking::default","modelScoreConfigSlug":"nemotron-cascade-14b-thinking","modelScoreStateName":"Nemotron Cascade 14B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":306.9625,"rankLo":259.975,"rankHi":354.0},{"slug":"deepseek-v3-0324","modelId":"deepseek-v3-0324","modelSlug":"deepseek-v3-0324","name":"DeepSeek-V3-0324","organizationSlug":"deepseek","n":67,"rankEligible":true,"score":135.06,"scoreLo":133.23,"scoreHi":136.95,"modelScoreStateId":"deepseek-v3-0324::default","modelScoreConfigSlug":"deepseek-v3-0324","modelScoreStateName":"DeepSeek-V3-0324","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":307.1128,"rankLo":287.0,"rankHi":328.0},{"slug":"palmyra-x5","modelId":"palmyra-x5","modelSlug":"palmyra-x5","name":"Palmyra X5","organizationSlug":"writer","n":9,"rankEligible":true,"score":135.05,"scoreLo":130.36,"scoreHi":139.98,"modelScoreStateId":"palmyra-x5::default","modelScoreConfigSlug":"palmyra-x5","modelScoreStateName":"Palmyra X5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":307.2492,"rankLo":244.0,"rankHi":367.0},{"slug":"mistral-medium-2508","modelId":"mistral-medium-2508","modelSlug":"mistral-medium-2508","name":"Mistral Medium 3.1","organizationSlug":"mistral-ai","n":36,"rankEligible":true,"score":135.03,"scoreLo":132.85,"scoreHi":137.26,"modelScoreStateId":"mistral-medium-2508::default","modelScoreConfigSlug":"mistral-medium-2508","modelScoreStateName":"Mistral Medium 3.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":307.6497,"rankLo":281.0,"rankHi":335.0},{"slug":"qwen3-32b","modelId":"qwen3-32b","modelSlug":"qwen3-32b","name":"Qwen3 32B","organizationSlug":"alibaba","n":37,"rankEligible":true,"score":134.96,"scoreLo":133.06,"scoreHi":136.79,"modelScoreStateId":"qwen3-32b::enabled","modelScoreConfigSlug":"qwen3-32b-thinking","modelScoreStateName":"Qwen3 32B (thinking)","modelScoreStateBestProbability":0.9936111111111111,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":308.5908,"rankLo":289.0,"rankHi":329.0},{"slug":"k2-think","modelId":"k2-think","modelSlug":"k2-think","name":"K2-Think","organizationSlug":"llm360","n":5,"rankEligible":true,"score":134.88,"scoreLo":130.94,"scoreHi":138.77,"modelScoreStateId":"k2-think::default","modelScoreConfigSlug":"k2-think","modelScoreStateName":"K2-Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":309.6661,"rankLo":260.0,"rankHi":358.0},{"slug":"gpt-oss-20b","modelId":"gpt-oss-20b","modelSlug":"gpt-oss-20b","name":"gpt-oss-20b","organizationSlug":"openai","n":43,"rankEligible":true,"score":134.84,"scoreLo":133.14,"scoreHi":136.58,"modelScoreStateId":"gpt-oss-20b::effort:high","modelScoreConfigSlug":"gpt-oss-20b-high","modelScoreStateName":"gpt-oss-20b (high)","modelScoreStateBestProbability":0.9811111111111112,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":310.1978,"rankLo":293.0,"rankHi":328.0},{"slug":"mistral-large-3-675b-instruct-2512","modelId":"mistral-large-3-675b-instruct-2512","modelSlug":"mistral-large-3-675b-instruct-2512","name":"Mistral Large 3 675B Instruct 2512","organizationSlug":"mistral-ai","n":68,"rankEligible":true,"score":134.68,"scoreLo":132.67,"scoreHi":136.68,"modelScoreStateId":"mistral-large-3-675b-instruct-2512::default","modelScoreConfigSlug":"mistral-large-3-675b-instruct-2512","modelScoreStateName":"Mistral Large 3 675B Instruct 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":312.5467,"rankLo":291.0,"rankHi":335.025},{"slug":"qwen3-omni-30b-a3b-thinking","modelId":"qwen3-omni-30b-a3b-thinking","modelSlug":"qwen3-omni-30b-a3b-thinking","name":"Qwen3 Omni 30B A3B Thinking","organizationSlug":"alibaba","n":30,"rankEligible":true,"score":134.67,"scoreLo":132.18,"scoreHi":137.17,"modelScoreStateId":"qwen3-omni-30b-a3b-thinking::default","modelScoreConfigSlug":"qwen3-omni-30b-a3b-thinking","modelScoreStateName":"Qwen3 Omni 30B A3B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":312.7017,"rankLo":284.0,"rankHi":342.0},{"slug":"solar-pro2-250710","modelId":"solar-pro2-250710","modelSlug":"solar-pro2-250710","name":"Solar Pro 2","organizationSlug":"upstage","n":8,"rankEligible":true,"score":134.49,"scoreLo":131.2,"scoreHi":137.8,"modelScoreStateId":"solar-pro2-250710::enabled","modelScoreConfigSlug":"solar-pro2-250710-thinking","modelScoreStateName":"Solar Pro 2 (thinking)","modelScoreStateBestProbability":0.9997222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":315.1617,"rankLo":273.0,"rankHi":355.0},{"slug":"qwen3-30b-a3b-instruct-2507","modelId":"qwen3-30b-a3b-instruct-2507","modelSlug":"qwen3-30b-a3b-instruct-2507","name":"Qwen3 30B A3B Instruct 2507","organizationSlug":"alibaba","n":38,"rankEligible":true,"score":134.42,"scoreLo":132.28,"scoreHi":136.48,"modelScoreStateId":"qwen3-30b-a3b-instruct-2507::default","modelScoreConfigSlug":"qwen3-30b-a3b-2507","modelScoreStateName":"Qwen3 30B A3B Instruct 2507","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":316.1967,"rankLo":292.0,"rankHi":340.0},{"slug":"ling-2-6-flash","modelId":"ling-2-6-flash","modelSlug":"ling-2-6-flash","name":"Ling 2.6 Flash","organizationSlug":"inclusion-ai","n":11,"rankEligible":true,"score":134.41,"scoreLo":129.45,"scoreHi":138.62,"modelScoreStateId":"ling-2-6-flash::default","modelScoreConfigSlug":"ling-2-6-flash","modelScoreStateName":"Ling 2.6 Flash","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":315.9125,"rankLo":261.0,"rankHi":382.0},{"slug":"llama-4-maverick-17b-128e-instruct-fp8","modelId":"llama-4-maverick-17b-128e-instruct-fp8","modelSlug":"llama-4-maverick-17b-128e-instruct-fp8","name":"Llama 4 Maverick Instruct FP8","organizationSlug":"meta","n":11,"rankEligible":true,"score":134.38,"scoreLo":130.85,"scoreHi":137.68,"modelScoreStateId":"llama-4-maverick-17b-128e-instruct-fp8::default","modelScoreConfigSlug":"llama-4-maverick-17b-128e-instruct-fp8","modelScoreStateName":"Llama 4 Maverick Instruct FP8","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":316.7464,"rankLo":275.975,"rankHi":358.0},{"slug":"k2-think-v2","modelId":"k2-think-v2","modelSlug":"k2-think-v2","name":"K2 Think V2","organizationSlug":"llm360","n":10,"rankEligible":true,"score":134.26,"scoreLo":129.85,"scoreHi":138.25,"modelScoreStateId":"k2-think-v2::effort:high","modelScoreConfigSlug":"k2-think-v2","modelScoreStateName":"K2 Think V2 (high)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":318.0997,"rankLo":267.0,"rankHi":373.025},{"slug":"apriel-1-5-15b-thinker","modelId":"apriel-1-5-15b-thinker","modelSlug":"apriel-1-5-15b-thinker","name":"Apriel-1.5-15B-Thinker","organizationSlug":"servicenow","n":9,"rankEligible":true,"score":134.2,"scoreLo":130.14,"scoreHi":138.22,"modelScoreStateId":"apriel-1-5-15b-thinker::default","modelScoreConfigSlug":"apriel-1-5-15b-thinker","modelScoreStateName":"Apriel-1.5-15B-Thinker","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":318.9336,"rankLo":267.0,"rankHi":369.0},{"slug":"k2-v2-instruct","modelId":"k2-v2-instruct","modelSlug":"k2-v2-instruct","name":"K2-V2-Instruct","organizationSlug":"llm360","n":8,"rankEligible":true,"score":134.19,"scoreLo":130.89,"scoreHi":137.31,"modelScoreStateId":"k2-v2-instruct::effort:high","modelScoreConfigSlug":"k2-v2-high","modelScoreStateName":"K2-V2-Instruct (high)","modelScoreStateBestProbability":0.8427777777777777,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":319.1367,"rankLo":279.0,"rankHi":359.0},{"slug":"exaone-4-0-32b","modelId":"exaone-4-0-32b","modelSlug":"exaone-4-0-32b","name":"EXAONE 4.0 32B","organizationSlug":"lg-ai-research","n":7,"rankEligible":true,"score":134.18,"scoreLo":130.63,"scoreHi":137.61,"modelScoreStateId":"exaone-4-0-32b::enabled","modelScoreConfigSlug":"exaone-4-0-32b-thinking","modelScoreStateName":"EXAONE 4.0 32B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":319.3467,"rankLo":275.975,"rankHi":363.0},{"slug":"phi-4-reasoning-plus","modelId":"phi-4-reasoning-plus","modelSlug":"phi-4-reasoning-plus","name":"Phi-4-reasoning-plus","organizationSlug":"microsoft","n":6,"rankEligible":true,"score":134.18,"scoreLo":129.47,"scoreHi":138.95,"modelScoreStateId":"phi-4-reasoning-plus::default","modelScoreConfigSlug":"phi-4-reasoning-plus","modelScoreStateName":"Phi-4-reasoning-plus","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":319.1222,"rankLo":258.0,"rankHi":377.0},{"slug":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","modelId":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","modelSlug":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","name":"Nemotron 3 Nano Omni 30B A3B","organizationSlug":"nvidia","n":16,"rankEligible":true,"score":134.01,"scoreLo":130.58,"scoreHi":137.37,"modelScoreStateId":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16::default","modelScoreConfigSlug":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","modelScoreStateName":"Nemotron 3 Nano Omni 30B A3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":321.4936,"rankLo":280.0,"rankHi":363.0},{"slug":"hyperclovax-seed-think-32b","modelId":"hyperclovax-seed-think-32b","modelSlug":"hyperclovax-seed-think-32b","name":"HyperCLOVA X SEED 32B Think","organizationSlug":"naver","n":8,"rankEligible":true,"score":133.82,"scoreLo":129.37,"scoreHi":138.15,"modelScoreStateId":"hyperclovax-seed-think-32b::default","modelScoreConfigSlug":"hyperclovax-seed-think-32b","modelScoreStateName":"HyperCLOVA X SEED 32B Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":323.9525,"rankLo":268.0,"rankHi":381.0},{"slug":"nemotron-nano-12b-v2","modelId":"nemotron-nano-12b-v2","modelSlug":"nemotron-nano-12b-v2","name":"Nemotron Nano 12B V2","organizationSlug":"nvidia","n":5,"rankEligible":true,"score":133.82,"scoreLo":128.55,"scoreHi":139.12,"modelScoreStateId":"nemotron-nano-12b-v2::enabled","modelScoreConfigSlug":"nemotron-nano-12b-v2-thinking","modelScoreStateName":"Nemotron Nano 12B V2 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":324.0117,"rankLo":256.0,"rankHi":391.025},{"slug":"qwen3-coder-30b-a3b-instruct","modelId":"qwen3-coder-30b-a3b-instruct","modelSlug":"qwen3-coder-30b-a3b-instruct","name":"Qwen3 Coder 30B A3B Instruct","organizationSlug":"alibaba","n":16,"rankEligible":true,"score":133.82,"scoreLo":130.58,"scoreHi":136.99,"modelScoreStateId":"qwen3-coder-30b-a3b-instruct::default","modelScoreConfigSlug":"qwen3-coder-30b-a3b-instruct","modelScoreStateName":"Qwen3 Coder 30B A3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":324.2075,"rankLo":285.0,"rankHi":363.0},{"slug":"nemotron-cascade-8b-thinking","modelId":"nemotron-cascade-8b-thinking","modelSlug":"nemotron-cascade-8b-thinking","name":"Nemotron Cascade 8B Thinking","organizationSlug":"nvidia","n":8,"rankEligible":true,"score":133.72,"scoreLo":129.65,"scoreHi":137.79,"modelScoreStateId":"nemotron-cascade-8b-thinking::default","modelScoreConfigSlug":"nemotron-cascade-8b-thinking","modelScoreStateName":"Nemotron Cascade 8B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":325.4597,"rankLo":272.975,"rankHi":376.0},{"slug":"g9v3-3b","modelId":"g9v3-3b","modelSlug":"g9v3-3b","name":"G9v3-3B","organizationSlug":"ai9stars","n":5,"rankEligible":true,"score":133.67,"scoreLo":127.98,"scoreHi":138.62,"modelScoreStateId":"g9v3-3b::default","modelScoreConfigSlug":"g9v3-3b","modelScoreStateName":"G9v3-3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":325.7986,"rankLo":261.975,"rankHi":399.0},{"slug":"qwq-32b","modelId":"qwq-32b","modelSlug":"qwq-32b","name":"QwQ-32B","organizationSlug":"alibaba","n":31,"rankEligible":true,"score":133.64,"scoreLo":131.23,"scoreHi":136.01,"modelScoreStateId":"qwq-32b::default","modelScoreConfigSlug":"qwq-32b","modelScoreStateName":"QwQ-32B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"unknown","modelScoreHarness":"none","rankMean":326.9036,"rankLo":298.0,"rankHi":354.0},{"slug":"qwen3-vl-8b-thinking","modelId":"qwen3-vl-8b-thinking","modelSlug":"qwen3-vl-8b-thinking","name":"Qwen3 VL 8B Thinking","organizationSlug":"alibaba","n":49,"rankEligible":true,"score":133.6,"scoreLo":131.35,"scoreHi":135.69,"modelScoreStateId":"qwen3-vl-8b-thinking::default","modelScoreConfigSlug":"qwen3-vl-8b-thinking","modelScoreStateName":"Qwen3 VL 8B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":327.6275,"rankLo":302.0,"rankHi":351.0},{"slug":"claude-3-5-sonnet-20241022","modelId":"claude-3-5-sonnet-20241022","modelSlug":"claude-3-5-sonnet-20241022","name":"Claude 3.5 Sonnet (Oct 2024)","organizationSlug":"anthropic","n":89,"rankEligible":true,"score":133.56,"scoreLo":131.61,"scoreHi":135.45,"modelScoreStateId":"claude-3-5-sonnet-20241022::default","modelScoreConfigSlug":"claude-3-5-sonnet-20241022","modelScoreStateName":"Claude 3.5 Sonnet (Oct 2024)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":328.2636,"rankLo":306.0,"rankHi":347.0},{"slug":"llama-3-1-nemotron-ultra-253b-v1","modelId":"llama-3-1-nemotron-ultra-253b-v1","modelSlug":"llama-3-1-nemotron-ultra-253b-v1","name":"Llama 3.1 Nemotron Ultra 253B V1","organizationSlug":"nvidia","n":11,"rankEligible":true,"score":133.55,"scoreLo":130.32,"scoreHi":136.51,"modelScoreStateId":"llama-3-1-nemotron-ultra-253b-v1::enabled","modelScoreConfigSlug":"llama-3-1-nemotron-ultra-253b-v1-thinking","modelScoreStateName":"Llama 3.1 Nemotron Ultra 253B V1 (thinking)","modelScoreStateBestProbability":0.9975,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":327.9536,"rankLo":290.0,"rankHi":364.0},{"slug":"qwen3-30b-a3b","modelId":"qwen3-30b-a3b","modelSlug":"qwen3-30b-a3b","name":"Qwen3-30B-A3B","organizationSlug":"alibaba","n":31,"rankEligible":true,"score":133.49,"scoreLo":131.57,"scoreHi":135.35,"modelScoreStateId":"qwen3-30b-a3b::enabled","modelScoreConfigSlug":"qwen3-30b-a3b-thinking","modelScoreStateName":"Qwen3-30B-A3B (thinking)","modelScoreStateBestProbability":0.9922222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":329.2158,"rankLo":309.0,"rankHi":347.025},{"slug":"gemini-2-0-pro-exp-02-05","modelId":"gemini-2-0-pro-exp-02-05","modelSlug":"gemini-2-0-pro-exp-02-05","name":"Gemini 2.0 Pro Experimental 02-05","organizationSlug":"google","n":17,"rankEligible":true,"score":133.48,"scoreLo":130.45,"scoreHi":136.43,"modelScoreStateId":"gemini-2-0-pro-exp-02-05::default","modelScoreConfigSlug":"gemini-2-0-pro-exp-feb-2025","modelScoreStateName":"Gemini 2.0 Pro Experimental 02-05","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":328.9008,"rankLo":291.0,"rankHi":364.0},{"slug":"gpt-4-1-mini","modelId":"gpt-4-1-mini","modelSlug":"gpt-4-1-mini","name":"GPT-4.1 Mini","organizationSlug":"openai","n":88,"rankEligible":true,"score":133.45,"scoreLo":131.56,"scoreHi":135.24,"modelScoreStateId":"gpt-4-1-mini::default","modelScoreConfigSlug":"gpt-4-1-mini","modelScoreStateName":"GPT-4.1 Mini","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":329.6925,"rankLo":310.0,"rankHi":348.0},{"slug":"qwen3-14b","modelId":"qwen3-14b","modelSlug":"qwen3-14b","name":"Qwen3 14B","organizationSlug":"alibaba","n":32,"rankEligible":true,"score":133.44,"scoreLo":131.26,"scoreHi":135.47,"modelScoreStateId":"qwen3-14b::enabled","modelScoreConfigSlug":"qwen3-14b-thinking","modelScoreStateName":"Qwen3 14B (thinking)","modelScoreStateBestProbability":0.9813888888888889,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":329.7292,"rankLo":306.0,"rankHi":352.0},{"slug":"gemini-2-5-flash-lite","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite","organizationSlug":"google","n":8,"rankEligible":true,"score":133.38,"scoreLo":130.94,"scoreHi":135.68,"modelScoreStateId":"gemini-2-5-flash-lite::effort:high","modelScoreConfigSlug":"gemini-2-5-flash-lite-high","modelScoreStateName":"Gemini 2.5 Flash-Lite (high)","modelScoreStateBestProbability":0.4494444444444444,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":330.5278,"rankLo":304.0,"rankHi":356.0},{"slug":"nemotron-h-47b-reasoning-128k","modelId":"nemotron-h-47b-reasoning-128k","modelSlug":"nemotron-h-47b-reasoning-128k","name":"Nemotron-H 47B Reasoning 128K","organizationSlug":"nvidia","n":5,"rankEligible":true,"score":133.37,"scoreLo":128.12,"scoreHi":138.65,"modelScoreStateId":"nemotron-h-47b-reasoning-128k::enabled","modelScoreConfigSlug":"nemotron-h-47b-reasoning-128k-thinking","modelScoreStateName":"Nemotron-H 47B Reasoning 128K (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":330.0022,"rankLo":259.975,"rankHi":399.0},{"slug":"internvl3-78b","modelId":"internvl3-78b","modelSlug":"internvl3-78b","name":"InternVL3-78B","organizationSlug":"opengvlab","n":9,"rankEligible":true,"score":133.2,"scoreLo":128.64,"scoreHi":137.97,"modelScoreStateId":"internvl3-78b::default","modelScoreConfigSlug":"internvl3-78b","modelScoreStateName":"InternVL3-78B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":332.3853,"rankLo":270.0,"rankHi":390.0},{"slug":"gemini-2-0-flash-thinking-exp-01-21","modelId":"gemini-2-0-flash-thinking-exp-01-21","modelSlug":"gemini-2-0-flash-thinking-exp-01-21","name":"Gemini 2.0 Flash Thinking Experimental 01-21","organizationSlug":"google","n":20,"rankEligible":true,"score":133.14,"scoreLo":130.23,"scoreHi":136.1,"modelScoreStateId":"gemini-2-0-flash-thinking-exp-01-21::default","modelScoreConfigSlug":"gemini-2-0-flash-thinking-exp","modelScoreStateName":"Gemini 2.0 Flash Thinking Experimental 01-21","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":333.4172,"rankLo":297.0,"rankHi":368.0},{"slug":"medgemma-27b-it","modelId":"medgemma-27b-it","modelSlug":"medgemma-27b-it","name":"MedGemma 27B IT","organizationSlug":"google","n":4,"rankEligible":true,"score":133.03,"scoreLo":124.68,"scoreHi":140.77,"modelScoreStateId":"medgemma-27b-it::default","modelScoreConfigSlug":"medgemma-27b-it","modelScoreStateName":"MedGemma 27B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":334.0025,"rankLo":229.0,"rankHi":440.0},{"slug":"minimax-m1-40k","modelId":"minimax-m1-40k","modelSlug":"minimax-m1-40k","name":"MiniMax M1 40k","organizationSlug":"minimax","n":5,"rankEligible":true,"score":132.92,"scoreLo":127.66,"scoreHi":137.62,"modelScoreStateId":"minimax-m1-40k::effort:40k","modelScoreConfigSlug":"minimax-m1-40k","modelScoreStateName":"MiniMax M1 40k (40k)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:40k","modelScoreReasoningEffort":"40k","modelScoreHarness":"none","rankMean":335.9994,"rankLo":276.0,"rankHi":401.0},{"slug":"longcat-flash-lite","modelId":"longcat-flash-lite","modelSlug":"longcat-flash-lite","name":"LongCat-Flash-Lite","organizationSlug":"meituan","n":5,"rankEligible":true,"score":132.86,"scoreLo":125.23,"scoreHi":139.1,"modelScoreStateId":"longcat-flash-lite::default","modelScoreConfigSlug":"longcat-flash-lite","modelScoreStateName":"LongCat-Flash-Lite","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":336.3386,"rankLo":255.0,"rankHi":429.025},{"slug":"ring-flash-2-0","modelId":"ring-flash-2-0","modelSlug":"ring-flash-2-0","name":"Ring-flash-2.0","organizationSlug":"inclusion-ai","n":8,"rankEligible":true,"score":132.75,"scoreLo":128.25,"scoreHi":137.01,"modelScoreStateId":"ring-flash-2-0::default","modelScoreConfigSlug":"ring-flash-2-0","modelScoreStateName":"Ring-flash-2.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":338.3522,"rankLo":284.975,"rankHi":394.0},{"slug":"solar-open-100b","modelId":"solar-open-100b","modelSlug":"solar-open-100b","name":"Solar Open 100B","organizationSlug":"upstage","n":6,"rankEligible":true,"score":132.69,"scoreLo":127.72,"scoreHi":137.4,"modelScoreStateId":"solar-open-100b::default","modelScoreConfigSlug":"solar-open-100b-thinking","modelScoreStateName":"Solar Open 100B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":339.1322,"rankLo":279.0,"rankHi":402.0},{"slug":"glm-4-5-air-fp8","modelId":"glm-4-5-air-fp8","modelSlug":"glm-4-5-air-fp8","name":"GLM 4.5 Air FP8","organizationSlug":"zhipu-ai","n":5,"rankEligible":true,"score":132.6,"scoreLo":127.24,"scoreHi":137.68,"modelScoreStateId":"glm-4-5-air-fp8::default","modelScoreConfigSlug":"glm-4-5-air-fp8-helm-capabilities","modelScoreStateName":"GLM 4.5 Air FP8","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":340.2117,"rankLo":276.0,"rankHi":404.025},{"slug":"qwen3-4b-thinking-2507","modelId":"qwen3-4b-thinking-2507","modelSlug":"qwen3-4b-thinking-2507","name":"Qwen3 4B Thinking 2507","organizationSlug":"alibaba","n":41,"rankEligible":true,"score":132.28,"scoreLo":130.07,"scoreHi":134.36,"modelScoreStateId":"qwen3-4b-thinking-2507::default","modelScoreConfigSlug":"qwen3-4b-2507-thinking","modelScoreStateName":"Qwen3 4B Thinking 2507","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":344.9158,"rankLo":323.0,"rankHi":366.0},{"slug":"o1-mini","modelId":"o1-mini","modelSlug":"o1-mini","name":"o1-mini","organizationSlug":"openai","n":39,"rankEligible":true,"score":132.14,"scoreLo":129.73,"scoreHi":134.55,"modelScoreStateId":"o1-mini::default","modelScoreConfigSlug":"o1-mini","modelScoreStateName":"o1-mini","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":346.6514,"rankLo":321.0,"rankHi":373.0},{"slug":"ling-flash-2-0","modelId":"ling-flash-2-0","modelSlug":"ling-flash-2-0","name":"Ling Flash 2.0","organizationSlug":"inclusion-ai","n":10,"rankEligible":true,"score":132.12,"scoreLo":127.62,"scoreHi":136.25,"modelScoreStateId":"ling-flash-2-0::default","modelScoreConfigSlug":"ling-flash-2-0","modelScoreStateName":"Ling Flash 2.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":346.7458,"rankLo":294.0,"rankHi":401.0},{"slug":"deepseek-v4-pro-base","modelId":"deepseek-v4-pro-base","modelSlug":"deepseek-v4-pro-base","name":"DeepSeek-V4-Pro-Base","organizationSlug":"deepseek","n":19,"rankEligible":true,"score":131.97,"scoreLo":128.97,"scoreHi":135.0,"modelScoreStateId":"deepseek-v4-pro-base::default","modelScoreConfigSlug":"deepseek-v4-pro-base","modelScoreStateName":"DeepSeek-V4-Pro-Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":348.8444,"rankLo":312.0,"rankHi":384.0},{"slug":"qwen3-omni-30b-a3b-instruct","modelId":"qwen3-omni-30b-a3b-instruct","modelSlug":"qwen3-omni-30b-a3b-instruct","name":"Qwen3 Omni 30B A3B Instruct","organizationSlug":"alibaba","n":26,"rankEligible":true,"score":131.88,"scoreLo":129.16,"scoreHi":134.48,"modelScoreStateId":"qwen3-omni-30b-a3b-instruct::default","modelScoreConfigSlug":"qwen3-omni-30b-a3b-instruct","modelScoreStateName":"Qwen3 Omni 30B A3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":349.9928,"rankLo":320.975,"rankHi":380.025},{"slug":"celeris-1","modelId":"celeris-1","modelSlug":"celeris-1","name":"Celeris-1","organizationSlug":"celeris","n":12,"rankEligible":true,"score":131.65,"scoreLo":127.77,"scoreHi":135.02,"modelScoreStateId":"celeris-1::default","modelScoreConfigSlug":"celeris-1","modelScoreStateName":"Celeris-1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":352.95,"rankLo":311.0,"rankHi":399.0},{"slug":"internvl3-38b","modelId":"internvl3-38b","modelSlug":"internvl3-38b","name":"InternVL3-38B","organizationSlug":"opengvlab","n":7,"rankEligible":true,"score":131.59,"scoreLo":126.87,"scoreHi":136.4,"modelScoreStateId":"internvl3-38b::default","modelScoreConfigSlug":"internvl3-38b","modelScoreStateName":"InternVL3-38B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":353.6783,"rankLo":292.0,"rankHi":410.0},{"slug":"deepseek-v3","modelId":"deepseek-v3","modelSlug":"deepseek-v3","name":"DeepSeek-V3","organizationSlug":"deepseek","n":84,"rankEligible":true,"score":131.56,"scoreLo":129.54,"scoreHi":133.48,"modelScoreStateId":"deepseek-v3::default","modelScoreConfigSlug":"deepseek-v3","modelScoreStateName":"DeepSeek-V3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":353.8836,"rankLo":335.0,"rankHi":374.0},{"slug":"ministral-3-8b-reasoning-2512","modelId":"ministral-3-8b-reasoning-2512","modelSlug":"ministral-3-8b-reasoning-2512","name":"Ministral 3 8B Reasoning 2512","organizationSlug":"mistral-ai","n":7,"rankEligible":true,"score":131.56,"scoreLo":127.2,"scoreHi":135.8,"modelScoreStateId":"ministral-3-8b-reasoning-2512::default","modelScoreConfigSlug":"ministral-3-8b-reasoning-2512","modelScoreStateName":"Ministral 3 8B Reasoning 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":354.3069,"rankLo":302.0,"rankHi":405.025},{"slug":"gemma-4-e4b-it","modelId":"gemma-4-e4b-it","modelSlug":"gemma-4-e4b-it","name":"Gemma 4 E4B IT","organizationSlug":"google","n":29,"rankEligible":true,"score":131.52,"scoreLo":129.18,"scoreHi":133.7,"modelScoreStateId":"gemma-4-e4b-it::enabled","modelScoreConfigSlug":"gemma-4-e4b-it-thinking","modelScoreStateName":"Gemma 4 E4B IT (thinking)","modelScoreStateBestProbability":0.7663888888888889,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":354.5639,"rankLo":332.975,"rankHi":380.0},{"slug":"solar-pro3-260126","modelId":"solar-pro3-260126","modelSlug":"solar-pro3-260126","name":"Solar Pro 3","organizationSlug":"upstage","n":18,"rankEligible":true,"score":131.29,"scoreLo":127.42,"scoreHi":134.93,"modelScoreStateId":"solar-pro3-260126::default","modelScoreConfigSlug":"solar-pro3-260126","modelScoreStateName":"Solar Pro 3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":357.6114,"rankLo":314.0,"rankHi":403.0},{"slug":"nova-premier-v1-0","modelId":"nova-premier-v1-0","modelSlug":"nova-premier-v1-0","name":"Nova Premier","organizationSlug":"amazon","n":23,"rankEligible":true,"score":131.26,"scoreLo":128.03,"scoreHi":134.38,"modelScoreStateId":"nova-premier-v1-0::default","modelScoreConfigSlug":"nova-premier-v1-0","modelScoreStateName":"Nova Premier","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":358.2239,"rankLo":320.0,"rankHi":397.0},{"slug":"gemini-2-0-flash-exp","modelId":"gemini-2-0-flash-exp","modelSlug":"gemini-2-0-flash-exp","name":"Gemini 2.0 Flash Experimental","organizationSlug":"google","n":32,"rankEligible":true,"score":131.21,"scoreLo":128.75,"scoreHi":133.63,"modelScoreStateId":"gemini-2-0-flash-exp::default","modelScoreConfigSlug":"gemini-2-0-flash-exp","modelScoreStateName":"Gemini 2.0 Flash Experimental","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":358.6047,"rankLo":332.0,"rankHi":387.0},{"slug":"codestral-2508","modelId":"codestral-2508","modelSlug":"codestral-2508","name":"Codestral 25.08","organizationSlug":"mistral-ai","n":6,"rankEligible":true,"score":131.2,"scoreLo":124.28,"scoreHi":137.79,"modelScoreStateId":"codestral-2508::default","modelScoreConfigSlug":"codestral-2508","modelScoreStateName":"Codestral 25.08","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":358.3689,"rankLo":272.0,"rankHi":441.0},{"slug":"motif-2-12-7b-reasoning","modelId":"motif-2-12-7b-reasoning","modelSlug":"motif-2-12-7b-reasoning","name":"Motif-2-12.7B-Reasoning","organizationSlug":"motif","n":7,"rankEligible":true,"score":131.2,"scoreLo":126.4,"scoreHi":135.81,"modelScoreStateId":"motif-2-12-7b-reasoning::default","modelScoreConfigSlug":"motif-2-12-7b-reasoning","modelScoreStateName":"Motif-2-12.7B-Reasoning","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":358.7433,"rankLo":300.0,"rankHi":413.025},{"slug":"qwen2-5-vl-72b-instruct","modelId":"qwen2-5-vl-72b-instruct","modelSlug":"qwen2-5-vl-72b-instruct","name":"Qwen2.5-VL-72B-Instruct","organizationSlug":"alibaba","n":51,"rankEligible":true,"score":131.0,"scoreLo":128.41,"scoreHi":133.5,"modelScoreStateId":"qwen2-5-vl-72b-instruct::default","modelScoreConfigSlug":"qwen2-5-vl-72b-instruct","modelScoreStateName":"Qwen2.5-VL-72B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":361.4172,"rankLo":335.0,"rankHi":389.0},{"slug":"sonar-pro","modelId":"sonar-pro","modelSlug":"sonar-pro","name":"Sonar Pro","organizationSlug":"perplexity","n":5,"rankEligible":true,"score":130.99,"scoreLo":124.87,"scoreHi":137.03,"modelScoreStateId":"sonar-pro::default","modelScoreConfigSlug":"sonar-pro","modelScoreStateName":"Sonar Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":361.2425,"rankLo":281.975,"rankHi":433.0},{"slug":"gemini-2-0-flash","modelId":"gemini-2-0-flash","modelSlug":"gemini-2-0-flash","name":"Gemini 2.0 Flash","organizationSlug":"google","n":73,"rankEligible":true,"score":130.94,"scoreLo":128.9,"scoreHi":132.91,"modelScoreStateId":"gemini-2-0-flash::default","modelScoreConfigSlug":"gemini-2-0-flash","modelScoreStateName":"Gemini 2.0 Flash","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":362.0628,"rankLo":343.0,"rankHi":384.0},{"slug":"llama-3-3-nemotron-super-49b-v1","modelId":"llama-3-3-nemotron-super-49b-v1","modelSlug":"llama-3-3-nemotron-super-49b-v1","name":"Llama 3.3 Nemotron Super 49B V1","organizationSlug":"nvidia","n":17,"rankEligible":true,"score":130.78,"scoreLo":127.88,"scoreHi":133.48,"modelScoreStateId":"llama-3-3-nemotron-super-49b-v1::enabled","modelScoreConfigSlug":"llama-3-3-nemotron-super-49b-v1-thinking","modelScoreStateName":"Llama 3.3 Nemotron Super 49B V1 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":364.415,"rankLo":335.0,"rankHi":396.025},{"slug":"olmo-3-1-32b-think","modelId":"olmo-3-1-32b-think","modelSlug":"olmo-3-1-32b-think","name":"Olmo 3.1 32B Think","organizationSlug":"allen-ai","n":14,"rankEligible":true,"score":130.73,"scoreLo":126.55,"scoreHi":134.84,"modelScoreStateId":"olmo-3-1-32b-think::default","modelScoreConfigSlug":"olmo-3-1-32b-think","modelScoreStateName":"Olmo 3.1 32B Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":365.0725,"rankLo":314.0,"rankHi":411.0},{"slug":"kimi-linear-48b-a3b-instruct","modelId":"kimi-linear-48b-a3b-instruct","modelSlug":"kimi-linear-48b-a3b-instruct","name":"Kimi Linear 48B A3B Instruct","organizationSlug":"moonshot-ai","n":13,"rankEligible":true,"score":130.69,"scoreLo":126.39,"scoreHi":134.58,"modelScoreStateId":"kimi-linear-48b-a3b-instruct::default","modelScoreConfigSlug":"kimi-linear-48b-a3b-instruct","modelScoreStateName":"Kimi Linear 48B A3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":365.6392,"rankLo":317.0,"rankHi":415.0},{"slug":"devstral-medium-2507","modelId":"devstral-medium-2507","modelSlug":"devstral-medium-2507","name":"Devstral Medium 1.0","organizationSlug":"mistral-ai","n":11,"rankEligible":true,"score":130.68,"scoreLo":126.82,"scoreHi":134.46,"modelScoreStateId":"devstral-medium-2507::default","modelScoreConfigSlug":"devstral-medium-2507","modelScoreStateName":"Devstral Medium 1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":365.7164,"rankLo":320.0,"rankHi":410.0},{"slug":"magistral-small-2509","modelId":"magistral-small-2509","modelSlug":"magistral-small-2509","name":"Magistral Small 1.2","organizationSlug":"mistral-ai","n":29,"rankEligible":true,"score":130.55,"scoreLo":127.84,"scoreHi":133.21,"modelScoreStateId":"magistral-small-2509::default","modelScoreConfigSlug":"magistral-small-2509","modelScoreStateName":"Magistral Small 1.2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":367.5006,"rankLo":339.0,"rankHi":398.0},{"slug":"qwen3-vl-8b-instruct","modelId":"qwen3-vl-8b-instruct","modelSlug":"qwen3-vl-8b-instruct","name":"Qwen3-VL-8B-Instruct","organizationSlug":"alibaba","n":52,"rankEligible":true,"score":130.48,"scoreLo":128.12,"scoreHi":132.62,"modelScoreStateId":"qwen3-vl-8b-instruct::default","modelScoreConfigSlug":"qwen3-vl-8b-instruct","modelScoreStateName":"Qwen3-VL-8B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":368.4447,"rankLo":346.0,"rankHi":393.0},{"slug":"magistral-medium-2506","modelId":"magistral-medium-2506","modelSlug":"magistral-medium-2506","name":"Magistral Medium 1.0","organizationSlug":"mistral-ai","n":24,"rankEligible":true,"score":130.47,"scoreLo":127.34,"scoreHi":133.25,"modelScoreStateId":"magistral-medium-2506::default","modelScoreConfigSlug":"magistral-medium-2506","modelScoreStateName":"Magistral Medium 1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":368.6769,"rankLo":335.0,"rankHi":404.0},{"slug":"qwen-max-2025-01-25","modelId":"qwen-max-2025-01-25","modelSlug":"qwen-max-2025-01-25","name":"Qwen2.5-Max","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":130.23,"scoreLo":126.44,"scoreHi":133.74,"modelScoreStateId":"qwen-max-2025-01-25::default","modelScoreConfigSlug":"qwen-max-2025-01-25","modelScoreStateName":"Qwen2.5-Max","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":371.6394,"rankLo":330.0,"rankHi":412.0},{"slug":"qwen3-vl-4b-thinking","modelId":"qwen3-vl-4b-thinking","modelSlug":"qwen3-vl-4b-thinking","name":"Qwen3 VL 4B Thinking","organizationSlug":"alibaba","n":47,"rankEligible":true,"score":130.16,"scoreLo":127.74,"scoreHi":132.43,"modelScoreStateId":"qwen3-vl-4b-thinking::default","modelScoreConfigSlug":"qwen3-vl-4b-thinking","modelScoreStateName":"Qwen3 VL 4B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":372.8072,"rankLo":349.0,"rankHi":396.025},{"slug":"deepseek-r1-zero","modelId":"deepseek-r1-zero","modelSlug":"deepseek-r1-zero","name":"DeepSeek-R1-Zero","organizationSlug":"deepseek","n":17,"rankEligible":true,"score":130.13,"scoreLo":126.54,"scoreHi":133.53,"modelScoreStateId":"deepseek-r1-zero::default","modelScoreConfigSlug":"deepseek-r1-zero","modelScoreStateName":"DeepSeek-R1-Zero","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":373.0128,"rankLo":332.0,"rankHi":412.0},{"slug":"claude-3-5-sonnet-20240620","modelId":"claude-3-5-sonnet-20240620","modelSlug":"claude-3-5-sonnet-20240620","name":"Claude 3.5 Sonnet (June 2024)","organizationSlug":"anthropic","n":72,"rankEligible":true,"score":130.0,"scoreLo":130.0,"scoreHi":130.0,"modelScoreStateId":"claude-3-5-sonnet-20240620::default","modelScoreConfigSlug":"claude-3-5-sonnet-20240620","modelScoreStateName":"Claude 3.5 Sonnet (June 2024)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":375.3594,"rankLo":354.0,"rankHi":397.0},{"slug":"lfm3-vl-2b","modelId":"lfm3-vl-2b","modelSlug":"lfm3-vl-2b","name":"LFM3-VL-2B","organizationSlug":"liquid-ai","n":49,"rankEligible":true,"score":129.98,"scoreLo":127.37,"scoreHi":132.51,"modelScoreStateId":"lfm3-vl-2b::enabled","modelScoreConfigSlug":"lfm3-vl-2b-thinking","modelScoreStateName":"LFM3-VL-2B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":375.1811,"rankLo":347.0,"rankHi":402.0},{"slug":"qwen3-8b","modelId":"qwen3-8b","modelSlug":"qwen3-8b","name":"Qwen3 8B","organizationSlug":"alibaba","n":35,"rankEligible":true,"score":129.96,"scoreLo":127.75,"scoreHi":132.02,"modelScoreStateId":"qwen3-8b::enabled","modelScoreConfigSlug":"qwen3-8b-thinking","modelScoreStateName":"Qwen3 8B (thinking)","modelScoreStateBestProbability":0.9947222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":375.6375,"rankLo":354.0,"rankHi":397.0},{"slug":"nemotron-nano-12b-v2-vl-fp8","modelId":"nemotron-nano-12b-v2-vl-fp8","modelSlug":"nemotron-nano-12b-v2-vl-fp8","name":"Nemotron Nano 12B V2 VL FP8","organizationSlug":"nvidia","n":8,"rankEligible":true,"score":129.94,"scoreLo":125.86,"scoreHi":133.82,"modelScoreStateId":"nemotron-nano-12b-v2-vl-fp8::enabled","modelScoreConfigSlug":"nemotron-nano-12b-v2-vl-fp8-thinking","modelScoreStateName":"Nemotron Nano 12B V2 VL FP8 (thinking)","modelScoreStateBestProbability":0.9866666666666667,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":375.3958,"rankLo":329.0,"rankHi":419.0},{"slug":"qwen-vl-max-0809","modelId":"qwen-vl-max-0809","modelSlug":"qwen-vl-max-0809","name":"Qwen VL Max 0809","organizationSlug":"alibaba","n":4,"rankEligible":true,"score":129.9,"scoreLo":121.83,"scoreHi":138.38,"modelScoreStateId":"qwen-vl-max-0809::default","modelScoreConfigSlug":"qwen-vl-max-0809","modelScoreStateName":"Qwen VL Max 0809","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":374.5872,"rankLo":265.0,"rankHi":474.0},{"slug":"gpt-4o-2024-11-20","modelId":"gpt-4o-2024-11-20","modelSlug":"gpt-4o-2024-11-20","name":"GPT-4o (2024-11-20)","organizationSlug":"openai","n":74,"rankEligible":true,"score":129.81,"scoreLo":127.58,"scoreHi":131.86,"modelScoreStateId":"gpt-4o-2024-11-20::default","modelScoreConfigSlug":"gpt-4o-2024-11-20","modelScoreStateName":"GPT-4o (2024-11-20)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":377.705,"rankLo":357.0,"rankHi":399.025},{"slug":"ernie-4-5-300b-a47b-pt","modelId":"ernie-4-5-300b-a47b-pt","modelSlug":"ernie-4-5-300b-a47b-pt","name":"ERNIE 4.5 300B A47B","organizationSlug":"baidu","n":9,"rankEligible":true,"score":129.73,"scoreLo":124.54,"scoreHi":134.83,"modelScoreStateId":"ernie-4-5-300b-a47b-pt::default","modelScoreConfigSlug":"ernie-4-5-300b-a47b-pt","modelScoreStateName":"ERNIE 4.5 300B A47B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":377.8242,"rankLo":315.0,"rankHi":438.0},{"slug":"grok-2","modelId":"grok-2","modelSlug":"grok-2","name":"Grok 2","organizationSlug":"xai","n":12,"rankEligible":true,"score":129.71,"scoreLo":125.92,"scoreHi":133.38,"modelScoreStateId":"grok-2::default","modelScoreConfigSlug":"grok-2","modelScoreStateName":"Grok 2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":378.3906,"rankLo":333.975,"rankHi":420.0},{"slug":"internvl2-5-78b","modelId":"internvl2-5-78b","modelSlug":"internvl2-5-78b","name":"InternVL2.5-78B","organizationSlug":"opengvlab","n":21,"rankEligible":true,"score":129.67,"scoreLo":126.39,"scoreHi":132.67,"modelScoreStateId":"internvl2-5-78b::default","modelScoreConfigSlug":"internvl2-5-78b","modelScoreStateName":"InternVL2.5-78B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":379.1169,"rankLo":343.0,"rankHi":412.0},{"slug":"gpt-4o-2024-05-13","modelId":"gpt-4o-2024-05-13","modelSlug":"gpt-4o-2024-05-13","name":"GPT-4o (2024-05-13)","organizationSlug":"openai","n":59,"rankEligible":true,"score":129.59,"scoreLo":127.33,"scoreHi":131.72,"modelScoreStateId":"gpt-4o-2024-05-13::default","modelScoreConfigSlug":"gpt-4o-2024-05-13","modelScoreStateName":"GPT-4o (2024-05-13)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":380.6486,"rankLo":357.0,"rankHi":402.0},{"slug":"gpt-4o-2024-08-06","modelId":"gpt-4o-2024-08-06","modelSlug":"gpt-4o-2024-08-06","name":"GPT-4o (2024-08-06)","organizationSlug":"openai","n":73,"rankEligible":true,"score":129.58,"scoreLo":127.38,"scoreHi":131.56,"modelScoreStateId":"gpt-4o-2024-08-06::default","modelScoreConfigSlug":"gpt-4o-2024-08-06","modelScoreStateName":"GPT-4o (2024-08-06)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":380.8136,"rankLo":359.0,"rankHi":401.0},{"slug":"llama-4-maverick-17b-128e-instruct","modelId":"llama-4-maverick-17b-128e-instruct","modelSlug":"llama-4-maverick-17b-128e-instruct","name":"Llama 4 Maverick Instruct","organizationSlug":"meta","n":105,"rankEligible":true,"score":129.5,"scoreLo":127.48,"scoreHi":131.45,"modelScoreStateId":"llama-4-maverick-17b-128e-instruct::default","modelScoreConfigSlug":"llama-4-maverick-17b-128e-instruct","modelScoreStateName":"Llama 4 Maverick Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":382.0581,"rankLo":363.0,"rankHi":400.0},{"slug":"gemini-1-5-pro-002","modelId":"gemini-1-5-pro-002","modelSlug":"gemini-1-5-pro-002","name":"Gemini 1.5 Pro 002","organizationSlug":"google","n":55,"rankEligible":true,"score":129.42,"scoreLo":126.91,"scoreHi":131.64,"modelScoreStateId":"gemini-1-5-pro-002::default","modelScoreConfigSlug":"gemini-1-5-pro-002","modelScoreStateName":"Gemini 1.5 Pro 002","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":382.7183,"rankLo":357.0,"rankHi":406.0},{"slug":"nemotron-3-ultra-550b-a55b-base-bf16","modelId":"nemotron-3-ultra-550b-a55b-base-bf16","modelSlug":"nemotron-3-ultra-550b-a55b-base-bf16","name":"Nemotron 3 Ultra 550B A55B Base","organizationSlug":"nvidia","n":12,"rankEligible":true,"score":129.28,"scoreLo":125.51,"scoreHi":133.02,"modelScoreStateId":"nemotron-3-ultra-550b-a55b-base-bf16::default","modelScoreConfigSlug":"nemotron-3-ultra-550b-a55b-base-bf16","modelScoreStateName":"Nemotron 3 Ultra 550B A55B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":383.8853,"rankLo":339.0,"rankHi":423.0},{"slug":"deepseek-v4-flash-base","modelId":"deepseek-v4-flash-base","modelSlug":"deepseek-v4-flash-base","name":"DeepSeek-V4-Flash-Base","organizationSlug":"deepseek","n":19,"rankEligible":true,"score":129.24,"scoreLo":126.19,"scoreHi":132.34,"modelScoreStateId":"deepseek-v4-flash-base::default","modelScoreConfigSlug":"deepseek-v4-flash-base","modelScoreStateName":"DeepSeek-V4-Flash-Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":384.8044,"rankLo":348.0,"rankHi":416.0},{"slug":"mistral-medium-2505","modelId":"mistral-medium-2505","modelSlug":"mistral-medium-2505","name":"Mistral Medium 3","organizationSlug":"mistral-ai","n":38,"rankEligible":true,"score":129.18,"scoreLo":126.48,"scoreHi":131.62,"modelScoreStateId":"mistral-medium-2505::default","modelScoreConfigSlug":"mistral-medium-2505","modelScoreStateName":"Mistral Medium 3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":385.8858,"rankLo":358.0,"rankHi":411.0},{"slug":"gemini-1-5-pro-exp-0827","modelId":"gemini-1-5-pro-exp-0827","modelSlug":"gemini-1-5-pro-exp-0827","name":"Gemini 1.5 Pro Experimental 0827","organizationSlug":"google","n":11,"rankEligible":true,"score":129.13,"scoreLo":125.48,"scoreHi":132.95,"modelScoreStateId":"gemini-1-5-pro-exp-0827::default","modelScoreConfigSlug":"gemini-1-5-pro-exp-0827","modelScoreStateName":"Gemini 1.5 Pro Experimental 0827","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":385.9561,"rankLo":341.0,"rankHi":424.025},{"slug":"magistral-small-2506","modelId":"magistral-small-2506","modelSlug":"magistral-small-2506","name":"Magistral Small 1.0","organizationSlug":"mistral-ai","n":16,"rankEligible":true,"score":129.02,"scoreLo":125.38,"scoreHi":132.42,"modelScoreStateId":"magistral-small-2506::default","modelScoreConfigSlug":"magistral-small-2506","modelScoreStateName":"Magistral Small 1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":387.4236,"rankLo":349.0,"rankHi":423.0},{"slug":"olmo-3-32b-think","modelId":"olmo-3-32b-think","modelSlug":"olmo-3-32b-think","name":"Olmo 3 32B Think","organizationSlug":"allen-ai","n":12,"rankEligible":true,"score":128.83,"scoreLo":124.36,"scoreHi":132.93,"modelScoreStateId":"olmo-3-32b-think::default","modelScoreConfigSlug":"olmo-3-32b-think","modelScoreStateName":"Olmo 3 32B Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":389.3608,"rankLo":339.0,"rankHi":439.0},{"slug":"nemotron-nano-9b-v2","modelId":"nemotron-nano-9b-v2","modelSlug":"nemotron-nano-9b-v2","name":"Nemotron Nano 9B V2","organizationSlug":"nvidia","n":13,"rankEligible":true,"score":128.82,"scoreLo":125.44,"scoreHi":132.07,"modelScoreStateId":"nemotron-nano-9b-v2::enabled","modelScoreConfigSlug":"nemotron-nano-9b-v2-thinking","modelScoreStateName":"Nemotron Nano 9B V2 (thinking)","modelScoreStateBestProbability":0.7277777777777777,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":389.9669,"rankLo":353.0,"rankHi":423.0},{"slug":"ministral-3-14b-instruct-2512","modelId":"ministral-3-14b-instruct-2512","modelSlug":"ministral-3-14b-instruct-2512","name":"Ministral 3 14B Instruct 2512","organizationSlug":"mistral-ai","n":29,"rankEligible":true,"score":128.79,"scoreLo":125.76,"scoreHi":131.61,"modelScoreStateId":"ministral-3-14b-instruct-2512::default","modelScoreConfigSlug":"ministral-3-14b-instruct-2512","modelScoreStateName":"Ministral 3 14B Instruct 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":390.6,"rankLo":359.975,"rankHi":419.0},{"slug":"internvl3-14b","modelId":"internvl3-14b","modelSlug":"internvl3-14b","name":"InternVL3-14B","organizationSlug":"opengvlab","n":7,"rankEligible":true,"score":128.65,"scoreLo":123.73,"scoreHi":133.28,"modelScoreStateId":"internvl3-14b::default","modelScoreConfigSlug":"internvl3-14b","modelScoreStateName":"InternVL3-14B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":391.5078,"rankLo":334.0,"rankHi":446.0},{"slug":"granite-4-2-3b","modelId":"granite-4-2-3b","modelSlug":"granite-4-2-3b","name":"Granite 4.2 3B","organizationSlug":"ibm","n":11,"rankEligible":true,"score":128.63,"scoreLo":123.86,"scoreHi":132.92,"modelScoreStateId":"granite-4-2-3b::enabled","modelScoreConfigSlug":"granite-4-2-3b-thinking","modelScoreStateName":"Granite 4.2 3B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":391.7586,"rankLo":340.0,"rankHi":444.0},{"slug":"hermes-4-70b","modelId":"hermes-4-70b","modelSlug":"hermes-4-70b","name":"Hermes 4 70B","organizationSlug":"nous-research","n":7,"rankEligible":true,"score":128.51,"scoreLo":124.32,"scoreHi":132.45,"modelScoreStateId":"hermes-4-70b::enabled","modelScoreConfigSlug":"hermes-4-70b-thinking","modelScoreStateName":"Hermes 4 70B (thinking)","modelScoreStateBestProbability":0.9363888888888889,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":393.4169,"rankLo":347.0,"rankHi":438.0},{"slug":"nemotron-3-super-120b-a12b-base-bf16","modelId":"nemotron-3-super-120b-a12b-base-bf16","modelSlug":"nemotron-3-super-120b-a12b-base-bf16","name":"Nemotron 3 Super 120B A12B Base","organizationSlug":"nvidia","n":11,"rankEligible":true,"score":128.44,"scoreLo":124.09,"scoreHi":132.7,"modelScoreStateId":"nemotron-3-super-120b-a12b-base-bf16::default","modelScoreConfigSlug":"nemotron-3-super-120b-a12b-base-bf16","modelScoreStateName":"Nemotron 3 Super 120B A12B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":394.22,"rankLo":343.0,"rankHi":440.0},{"slug":"nemotron-nano-12b-v2-vl-bf16","modelId":"nemotron-nano-12b-v2-vl-bf16","modelSlug":"nemotron-nano-12b-v2-vl-bf16","name":"Nemotron Nano 12B V2 VL BF16","organizationSlug":"nvidia","n":18,"rankEligible":true,"score":128.2,"scoreLo":124.7,"scoreHi":131.34,"modelScoreStateId":"nemotron-nano-12b-v2-vl-bf16::enabled","modelScoreConfigSlug":"nemotron-nano-12b-v2-vl-bf16-thinking","modelScoreStateName":"Nemotron Nano 12B V2 VL BF16 (thinking)","modelScoreStateBestProbability":0.9741666666666666,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":397.4625,"rankLo":360.975,"rankHi":432.0},{"slug":"nemotron-3-nano-4b-bf16","modelId":"nemotron-3-nano-4b-bf16","modelSlug":"nemotron-3-nano-4b-bf16","name":"Nemotron 3 Nano 4B","organizationSlug":"nvidia","n":9,"rankEligible":true,"score":127.86,"scoreLo":124.42,"scoreHi":131.03,"modelScoreStateId":"nemotron-3-nano-4b-bf16::enabled","modelScoreConfigSlug":"nemotron-3-nano-4b-bf16-thinking","modelScoreStateName":"Nemotron 3 Nano 4B (thinking)","modelScoreStateBestProbability":0.9722222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":401.6436,"rankLo":366.0,"rankHi":435.0},{"slug":"grok-2-1212","modelId":"grok-2-1212","modelSlug":"grok-2-1212","name":"Grok 2 1212","organizationSlug":"xai","n":14,"rankEligible":true,"score":127.64,"scoreLo":124.37,"scoreHi":130.7,"modelScoreStateId":"grok-2-1212::default","modelScoreConfigSlug":"grok-2-1212","modelScoreStateName":"Grok 2 1212","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":404.2883,"rankLo":369.0,"rankHi":436.0},{"slug":"kimi-k2-base","modelId":"kimi-k2-base","modelSlug":"kimi-k2-base","name":"Kimi K2 Base","organizationSlug":"moonshot-ai","n":22,"rankEligible":true,"score":127.64,"scoreLo":124.47,"scoreHi":130.57,"modelScoreStateId":"kimi-k2-base::default","modelScoreConfigSlug":"kimi-k2-base","modelScoreStateName":"Kimi K2 Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":404.2667,"rankLo":371.0,"rankHi":434.0},{"slug":"gemini-2-0-flash-lite-preview-02-05","modelId":"gemini-2-0-flash-lite-preview-02-05","modelSlug":"gemini-2-0-flash-lite-preview-02-05","name":"Gemini 2.0 Flash-Lite Preview 02-05","organizationSlug":"google","n":15,"rankEligible":true,"score":127.61,"scoreLo":124.16,"scoreHi":130.74,"modelScoreStateId":"gemini-2-0-flash-lite-preview-02-05::default","modelScoreConfigSlug":"gemini-2-0-flash-lite-preview-02-05","modelScoreStateName":"Gemini 2.0 Flash-Lite Preview 02-05","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":404.5811,"rankLo":370.0,"rankHi":438.025},{"slug":"deepseek-r1-0528-qwen3-8b","modelId":"deepseek-r1-0528-qwen3-8b","modelSlug":"deepseek-r1-0528-qwen3-8b","name":"DeepSeek R1 0528 Qwen3 8B","organizationSlug":"deepseek","n":16,"rankEligible":true,"score":127.48,"scoreLo":123.61,"scoreHi":130.92,"modelScoreStateId":"deepseek-r1-0528-qwen3-8b::default","modelScoreConfigSlug":"deepseek-r1-0528-qwen3-8b","modelScoreStateName":"DeepSeek R1 0528 Qwen3 8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":406.1022,"rankLo":367.0,"rankHi":445.0},{"slug":"kimi-k1-5","modelId":"kimi-k1-5","modelSlug":"kimi-k1-5","name":"Kimi K1.5","organizationSlug":"moonshot-ai","n":4,"rankEligible":true,"score":127.42,"scoreLo":122.36,"scoreHi":132.32,"modelScoreStateId":"kimi-k1-5::default","modelScoreConfigSlug":"kimi-k1-5","modelScoreStateName":"Kimi K1.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":406.6094,"rankLo":348.0,"rankHi":465.0},{"slug":"tri-21b-think-preview","modelId":"tri-21b-think-preview","modelSlug":"tri-21b-think-preview","name":"Tri-21B-Think Preview","organizationSlug":"trillion-labs","n":5,"rankEligible":true,"score":127.42,"scoreLo":119.82,"scoreHi":133.95,"modelScoreStateId":"tri-21b-think-preview::default","modelScoreConfigSlug":"tri-21b-think-preview","modelScoreStateName":"Tri-21B-Think Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":406.6025,"rankLo":325.975,"rankHi":502.0},{"slug":"nemotron-h-8b-reasoning-128k","modelId":"nemotron-h-8b-reasoning-128k","modelSlug":"nemotron-h-8b-reasoning-128k","name":"Nemotron-H 8B Reasoning 128K","organizationSlug":"nvidia","n":5,"rankEligible":true,"score":127.29,"scoreLo":121.38,"scoreHi":132.78,"modelScoreStateId":"nemotron-h-8b-reasoning-128k::enabled","modelScoreConfigSlug":"nemotron-h-8b-reasoning-128k-thinking","modelScoreStateName":"Nemotron-H 8B Reasoning 128K (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":408.2256,"rankLo":341.0,"rankHi":480.025},{"slug":"qwen3-vl-4b-instruct","modelId":"qwen3-vl-4b-instruct","modelSlug":"qwen3-vl-4b-instruct","name":"Qwen3 VL 4B Instruct","organizationSlug":"alibaba","n":48,"rankEligible":true,"score":127.24,"scoreLo":124.64,"scoreHi":129.57,"modelScoreStateId":"qwen3-vl-4b-instruct::default","modelScoreConfigSlug":"qwen3-vl-4b-instruct","modelScoreStateName":"Qwen3 VL 4B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":409.0178,"rankLo":389.0,"rankHi":430.0},{"slug":"qwen2-5-vl-32b-instruct","modelId":"qwen2-5-vl-32b-instruct","modelSlug":"qwen2-5-vl-32b-instruct","name":"Qwen2.5 VL 32B Instruct","organizationSlug":"alibaba","n":24,"rankEligible":true,"score":127.09,"scoreLo":123.86,"scoreHi":129.89,"modelScoreStateId":"qwen2-5-vl-32b-instruct::default","modelScoreConfigSlug":"qwen2-5-vl-32b-instruct","modelScoreStateName":"Qwen2.5 VL 32B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":410.7164,"rankLo":382.0,"rankHi":440.0},{"slug":"devstral-small-2505","modelId":"devstral-small-2505","modelSlug":"devstral-small-2505","name":"Devstral Small 1.0","organizationSlug":"mistral-ai","n":8,"rankEligible":true,"score":127.05,"scoreLo":121.73,"scoreHi":131.91,"modelScoreStateId":"devstral-small-2505::default","modelScoreConfigSlug":"devstral-small-2505","modelScoreStateName":"Devstral Small 1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":411.2081,"rankLo":351.0,"rankHi":472.025},{"slug":"grok-2-mini","modelId":"grok-2-mini","modelSlug":"grok-2-mini","name":"Grok 2 Mini","organizationSlug":"xai","n":8,"rankEligible":true,"score":127.0,"scoreLo":122.74,"scoreHi":130.94,"modelScoreStateId":"grok-2-mini::default","modelScoreConfigSlug":"grok-2-mini","modelScoreStateName":"Grok 2 Mini","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":411.7264,"rankLo":365.975,"rankHi":456.0},{"slug":"mistral-small-3-2-24b-instruct-2506","modelId":"mistral-small-3-2-24b-instruct-2506","modelSlug":"mistral-small-3-2-24b-instruct-2506","name":"Mistral Small 3.2 24B Instruct 2506","organizationSlug":"mistral-ai","n":45,"rankEligible":true,"score":126.98,"scoreLo":124.22,"scoreHi":129.46,"modelScoreStateId":"mistral-small-3-2-24b-instruct-2506::default","modelScoreConfigSlug":"mistral-small-3-2-24b-instruct-2506","modelScoreStateName":"Mistral Small 3.2 24B Instruct 2506","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":411.9406,"rankLo":389.0,"rankHi":435.0},{"slug":"grok-beta","modelId":"grok-beta","modelSlug":"grok-beta","name":"Grok Beta","organizationSlug":"xai","n":7,"rankEligible":true,"score":126.94,"scoreLo":120.82,"scoreHi":132.81,"modelScoreStateId":"grok-beta::default","modelScoreConfigSlug":"grok-beta","modelScoreStateName":"Grok Beta","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":412.4953,"rankLo":340.0,"rankHi":483.025},{"slug":"qwen3-4b-instruct-2507","modelId":"qwen3-4b-instruct-2507","modelSlug":"qwen3-4b-instruct-2507","name":"Qwen3 4B Instruct 2507","organizationSlug":"alibaba","n":30,"rankEligible":true,"score":126.87,"scoreLo":124.07,"scoreHi":129.35,"modelScoreStateId":"qwen3-4b-instruct-2507::default","modelScoreConfigSlug":"qwen3-4b-2507","modelScoreStateName":"Qwen3 4B Instruct 2507","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":413.1597,"rankLo":391.0,"rankHi":437.0},{"slug":"pixtral-large-instruct-2411","modelId":"pixtral-large-instruct-2411","modelSlug":"pixtral-large-instruct-2411","name":"Pixtral Large","organizationSlug":"mistral-ai","n":15,"rankEligible":true,"score":126.66,"scoreLo":122.73,"scoreHi":130.37,"modelScoreStateId":"pixtral-large-instruct-2411::default","modelScoreConfigSlug":"pixtral-large-instruct-2411","modelScoreStateName":"Pixtral Large","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":415.8103,"rankLo":375.0,"rankHi":458.0},{"slug":"granite-4-1-8b","modelId":"granite-4-1-8b","modelSlug":"granite-4-1-8b","name":"Granite 4.1 8B","organizationSlug":"ibm","n":15,"rankEligible":true,"score":126.54,"scoreLo":122.02,"scoreHi":130.55,"modelScoreStateId":"granite-4-1-8b::default","modelScoreConfigSlug":"granite-4-1-8b","modelScoreStateName":"Granite 4.1 8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":417.2933,"rankLo":374.0,"rankHi":467.0},{"slug":"gpt-4-turbo-2024-04-09","modelId":"gpt-4-turbo-2024-04-09","modelSlug":"gpt-4-turbo-2024-04-09","name":"GPT-4 Turbo","organizationSlug":"openai","n":48,"rankEligible":true,"score":126.48,"scoreLo":123.96,"scoreHi":128.87,"modelScoreStateId":"gpt-4-turbo-2024-04-09::default","modelScoreConfigSlug":"gpt-4-turbo-2024-04-09","modelScoreStateName":"GPT-4 Turbo","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":417.5336,"rankLo":397.0,"rankHi":440.0},{"slug":"medgemma-27b-text-it","modelId":"medgemma-27b-text-it","modelSlug":"medgemma-27b-text-it","name":"MedGemma 27B Text IT","organizationSlug":"google","n":6,"rankEligible":true,"score":126.47,"scoreLo":120.43,"scoreHi":132.57,"modelScoreStateId":"medgemma-27b-text-it::default","modelScoreConfigSlug":"medgemma-27b-text-it","modelScoreStateName":"MedGemma 27B Text IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":418.3025,"rankLo":345.0,"rankHi":494.0},{"slug":"tri-21b-think","modelId":"tri-21b-think","modelSlug":"tri-21b-think","name":"Tri-21B-Think","organizationSlug":"trillion-labs","n":5,"rankEligible":true,"score":126.42,"scoreLo":119.9,"scoreHi":132.0,"modelScoreStateId":"tri-21b-think::default","modelScoreConfigSlug":"tri-21b-think","modelScoreStateName":"Tri-21B-Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":419.0467,"rankLo":349.0,"rankHi":500.0},{"slug":"grok-2-vision-1212","modelId":"grok-2-vision-1212","modelSlug":"grok-2-vision-1212","name":"Grok 2 Vision 1212","organizationSlug":"xai","n":5,"rankEligible":true,"score":126.4,"scoreLo":118.57,"scoreHi":134.27,"modelScoreStateId":"grok-2-vision-1212::default","modelScoreConfigSlug":"grok-2-vision-1212","modelScoreStateName":"Grok 2 Vision 1212","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":419.3381,"rankLo":321.0,"rankHi":517.0},{"slug":"qvq-72b-preview","modelId":"qvq-72b-preview","modelSlug":"qvq-72b-preview","name":"QVQ 72B Preview","organizationSlug":"alibaba","n":4,"rankEligible":true,"score":126.4,"scoreLo":121.17,"scoreHi":131.47,"modelScoreStateId":"qvq-72b-preview::default","modelScoreConfigSlug":"qvq-72b-preview","modelScoreStateName":"QVQ 72B Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":419.1322,"rankLo":358.0,"rankHi":479.0},{"slug":"mistral-large-instruct-2411","modelId":"mistral-large-instruct-2411","modelSlug":"mistral-large-instruct-2411","name":"Mistral Large 2.1 (Instruct 2411)","organizationSlug":"mistral-ai","n":32,"rankEligible":true,"score":126.1,"scoreLo":123.1,"scoreHi":128.8,"modelScoreStateId":"mistral-large-instruct-2411::default","modelScoreConfigSlug":"mistral-large-instruct-2411","modelScoreStateName":"Mistral Large 2.1 (Instruct 2411)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":422.115,"rankLo":397.0,"rankHi":450.0},{"slug":"kimi-vl-a3b-thinking-2506","modelId":"kimi-vl-a3b-thinking-2506","modelSlug":"kimi-vl-a3b-thinking-2506","name":"Kimi-VL-A3B-Thinking-2506","organizationSlug":"moonshot-ai","n":15,"rankEligible":true,"score":126.03,"scoreLo":122.57,"scoreHi":129.48,"modelScoreStateId":"kimi-vl-a3b-thinking-2506::default","modelScoreConfigSlug":"kimi-vl-a3b-thinking-2506-thinking","modelScoreStateName":"Kimi-VL-A3B-Thinking-2506","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":423.2161,"rankLo":390.0,"rankHi":459.0},{"slug":"sonar","modelId":"sonar","modelSlug":"sonar","name":"Sonar","organizationSlug":"perplexity","n":4,"rankEligible":true,"score":125.92,"scoreLo":119.55,"scoreHi":131.9,"modelScoreStateId":"sonar::default","modelScoreConfigSlug":"sonar","modelScoreStateName":"Sonar","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":425.3756,"rankLo":353.0,"rankHi":504.0},{"slug":"ministral-3-8b-instruct-2512","modelId":"ministral-3-8b-instruct-2512","modelSlug":"ministral-3-8b-instruct-2512","name":"Ministral 3 8B Instruct 2512","organizationSlug":"mistral-ai","n":28,"rankEligible":true,"score":125.88,"scoreLo":122.55,"scoreHi":128.91,"modelScoreStateId":"ministral-3-8b-instruct-2512::default","modelScoreConfigSlug":"ministral-3-8b-instruct-2512","modelScoreStateName":"Ministral 3 8B Instruct 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":424.865,"rankLo":396.0,"rankHi":457.0},{"slug":"reka-flash-3","modelId":"reka-flash-3","modelSlug":"reka-flash-3","name":"Reka Flash 3","organizationSlug":"reka-ai","n":11,"rankEligible":true,"score":125.78,"scoreLo":120.22,"scoreHi":131.0,"modelScoreStateId":"reka-flash-3::default","modelScoreConfigSlug":"reka-flash-3","modelScoreStateName":"Reka Flash 3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":426.8514,"rankLo":363.0,"rankHi":494.025},{"slug":"llama-3-1-405b-instruct","modelId":"llama-3-1-405b-instruct","modelSlug":"llama-3-1-405b-instruct","name":"Llama 3.1 405B Instruct","organizationSlug":"meta","n":85,"rankEligible":true,"score":125.77,"scoreLo":123.35,"scoreHi":127.93,"modelScoreStateId":"llama-3-1-405b-instruct::default","modelScoreConfigSlug":"llama-3-1-405b-instruct","modelScoreStateName":"Llama 3.1 405B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":425.8453,"rankLo":408.0,"rankHi":445.0},{"slug":"sarvam-30b","modelId":"sarvam-30b","modelSlug":"sarvam-30b","name":"Sarvam 30B","organizationSlug":"sarvam-ai","n":5,"rankEligible":true,"score":125.73,"scoreLo":116.13,"scoreHi":133.37,"modelScoreStateId":"sarvam-30b::effort:high","modelScoreConfigSlug":"sarvam-30b-high","modelScoreStateName":"Sarvam 30B (high)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":427.9997,"rankLo":332.0,"rankHi":548.025},{"slug":"qwen2-vl-72b-instruct","modelId":"qwen2-vl-72b-instruct","modelSlug":"qwen2-vl-72b-instruct","name":"Qwen2 VL 72B Instruct","organizationSlug":"alibaba","n":34,"rankEligible":true,"score":125.71,"scoreLo":122.6,"scoreHi":128.51,"modelScoreStateId":"qwen2-vl-72b-instruct::default","modelScoreConfigSlug":"qwen2-vl-72b-instruct","modelScoreStateName":"Qwen2 VL 72B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":426.7072,"rankLo":401.0,"rankHi":455.0},{"slug":"qwen2-5-72b-instruct","modelId":"qwen2-5-72b-instruct","modelSlug":"qwen2-5-72b-instruct","name":"Qwen2.5 72B Instruct","organizationSlug":"alibaba","n":61,"rankEligible":true,"score":125.5,"scoreLo":122.85,"scoreHi":127.89,"modelScoreStateId":"qwen2-5-72b-instruct::default","modelScoreConfigSlug":"qwen-2-5-72b-instruct","modelScoreStateName":"Qwen2.5 72B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":429.1925,"rankLo":409.0,"rankHi":451.0},{"slug":"deepseek-r1-distill-qwen-32b","modelId":"deepseek-r1-distill-qwen-32b","modelSlug":"deepseek-r1-distill-qwen-32b","name":"DeepSeek-R1-Distill-Qwen-32B","organizationSlug":"deepseek","n":33,"rankEligible":true,"score":125.33,"scoreLo":122.37,"scoreHi":128.02,"modelScoreStateId":"deepseek-r1-distill-qwen-32b::default","modelScoreConfigSlug":"deepseek-r1-distill-qwen-32b","modelScoreStateName":"DeepSeek-R1-Distill-Qwen-32B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":431.4314,"rankLo":407.0,"rankHi":459.0},{"slug":"llama-4-scout-17b-16e-instruct","modelId":"llama-4-scout-17b-16e-instruct","modelSlug":"llama-4-scout-17b-16e-instruct","name":"Llama 4 Scout Instruct","organizationSlug":"meta","n":87,"rankEligible":true,"score":125.29,"scoreLo":122.68,"scoreHi":127.56,"modelScoreStateId":"llama-4-scout-17b-16e-instruct::default","modelScoreConfigSlug":"llama-4-scout-17b-16e-instruct","modelScoreStateName":"Llama 4 Scout Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":431.7789,"rankLo":414.0,"rankHi":452.0},{"slug":"qwen3-4b","modelId":"qwen3-4b","modelSlug":"qwen3-4b","name":"Qwen3 4B","organizationSlug":"alibaba","n":25,"rankEligible":true,"score":125.17,"scoreLo":122.46,"scoreHi":127.73,"modelScoreStateId":"qwen3-4b::enabled","modelScoreConfigSlug":"qwen3-4b-thinking","modelScoreStateName":"Qwen3 4B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":433.315,"rankLo":412.0,"rankHi":457.0},{"slug":"deepseek-r1-distill-llama-70b","modelId":"deepseek-r1-distill-llama-70b","modelSlug":"deepseek-r1-distill-llama-70b","name":"DeepSeek-R1-Distill-Llama-70B","organizationSlug":"deepseek","n":30,"rankEligible":true,"score":125.16,"scoreLo":121.9,"scoreHi":128.01,"modelScoreStateId":"deepseek-r1-distill-llama-70b::default","modelScoreConfigSlug":"deepseek-r1-distill-llama-70b","modelScoreStateName":"DeepSeek-R1-Distill-Llama-70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":433.7142,"rankLo":406.0,"rankHi":465.0},{"slug":"mistral-small-3-1-24b-instruct-2503","modelId":"mistral-small-3-1-24b-instruct-2503","modelSlug":"mistral-small-3-1-24b-instruct-2503","name":"Mistral Small 3.1 24B Instruct 2503","organizationSlug":"mistral-ai","n":48,"rankEligible":true,"score":125.1,"scoreLo":122.08,"scoreHi":127.78,"modelScoreStateId":"mistral-small-3-1-24b-instruct-2503::default","modelScoreConfigSlug":"mistral-small-3-1-24b-instruct-2503","modelScoreStateName":"Mistral Small 3.1 24B Instruct 2503","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":434.265,"rankLo":409.0,"rankHi":461.025},{"slug":"gpt-4-0125-preview","modelId":"gpt-4-0125-preview","modelSlug":"gpt-4-0125-preview","name":"GPT-4 0125 Preview","organizationSlug":"openai","n":25,"rankEligible":true,"score":124.85,"scoreLo":121.68,"scoreHi":127.69,"modelScoreStateId":"gpt-4-0125-preview::default","modelScoreConfigSlug":"gpt-4-0125-preview","modelScoreStateName":"GPT-4 0125 Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":437.4772,"rankLo":410.0,"rankHi":468.0},{"slug":"ministral-3-3b-reasoning-2512","modelId":"ministral-3-3b-reasoning-2512","modelSlug":"ministral-3-3b-reasoning-2512","name":"Ministral 3 3B Reasoning 2512","organizationSlug":"mistral-ai","n":7,"rankEligible":true,"score":124.8,"scoreLo":119.64,"scoreHi":129.6,"modelScoreStateId":"ministral-3-3b-reasoning-2512::default","modelScoreConfigSlug":"ministral-3-3b-reasoning-2512","modelScoreStateName":"Ministral 3 3B Reasoning 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":438.8814,"rankLo":386.0,"rankHi":500.025},{"slug":"llama-3-1-nemotron-nano-8b-v1","modelId":"llama-3-1-nemotron-nano-8b-v1","modelSlug":"llama-3-1-nemotron-nano-8b-v1","name":"Llama 3.1 Nemotron Nano 8B V1","organizationSlug":"nvidia","n":6,"rankEligible":true,"score":124.73,"scoreLo":119.31,"scoreHi":129.77,"modelScoreStateId":"llama-3-1-nemotron-nano-8b-v1::enabled","modelScoreConfigSlug":"llama-3-1-nemotron-nano-8b-v1-thinking","modelScoreStateName":"Llama 3.1 Nemotron Nano 8B V1 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":440.0372,"rankLo":384.0,"rankHi":508.0},{"slug":"granite-4-1-30b","modelId":"granite-4-1-30b","modelSlug":"granite-4-1-30b","name":"Granite 4.1 30B","organizationSlug":"ibm","n":8,"rankEligible":true,"score":124.67,"scoreLo":117.44,"scoreHi":130.24,"modelScoreStateId":"granite-4-1-30b::default","modelScoreConfigSlug":"granite-4-1-30b","modelScoreStateName":"Granite 4.1 30B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":440.7778,"rankLo":374.0,"rankHi":526.0},{"slug":"gpt-4-1106-preview","modelId":"gpt-4-1106-preview","modelSlug":"gpt-4-1106-preview","name":"GPT-4 Turbo (1106 Preview)","organizationSlug":"openai","n":22,"rankEligible":true,"score":124.63,"scoreLo":121.25,"scoreHi":127.65,"modelScoreStateId":"gpt-4-1106-preview::default","modelScoreConfigSlug":"gpt-4-1106-preview","modelScoreStateName":"GPT-4 Turbo (1106 Preview)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":440.2672,"rankLo":410.0,"rankHi":473.0},{"slug":"claude-3-5-haiku-20241022","modelId":"claude-3-5-haiku-20241022","modelSlug":"claude-3-5-haiku-20241022","name":"Haiku 3.5","organizationSlug":"anthropic","n":70,"rankEligible":true,"score":124.59,"scoreLo":121.84,"scoreHi":127.03,"modelScoreStateId":"claude-3-5-haiku-20241022::default","modelScoreConfigSlug":"claude-3-5-haiku-20241022","modelScoreStateName":"Haiku 3.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":440.71,"rankLo":419.0,"rankHi":464.0},{"slug":"gemini-2-0-flash-lite","modelId":"gemini-2-0-flash-lite","modelSlug":"gemini-2-0-flash-lite","name":"Gemini 2.0 Flash-Lite","organizationSlug":"google","n":23,"rankEligible":true,"score":124.45,"scoreLo":121.0,"scoreHi":127.56,"modelScoreStateId":"gemini-2-0-flash-lite::default","modelScoreConfigSlug":"gemini-2-0-flash-lite","modelScoreStateName":"Gemini 2.0 Flash-Lite","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":442.7033,"rankLo":411.0,"rankHi":478.025},{"slug":"llama-3-3-70b-instruct","modelId":"llama-3-3-70b-instruct","modelSlug":"llama-3-3-70b-instruct","name":"Llama 3.3 70B Instruct","organizationSlug":"meta","n":67,"rankEligible":true,"score":124.45,"scoreLo":121.73,"scoreHi":126.97,"modelScoreStateId":"llama-3-3-70b-instruct::default","modelScoreConfigSlug":"llama-3-3-70b-instruct","modelScoreStateName":"Llama 3.3 70B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":442.3767,"rankLo":420.0,"rankHi":465.0},{"slug":"hermes-3-llama-3-1-405b","modelId":"hermes-3-llama-3-1-405b","modelSlug":"hermes-3-llama-3-1-405b","name":"Hermes 3 Llama 3.1 405B","organizationSlug":"nous-research","n":5,"rankEligible":true,"score":124.39,"scoreLo":118.3,"scoreHi":130.64,"modelScoreStateId":"hermes-3-llama-3-1-405b::default","modelScoreConfigSlug":"hermes-3-llama-3-1-405b","modelScoreStateName":"Hermes 3 Llama 3.1 405B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":444.4597,"rankLo":367.0,"rankHi":518.0},{"slug":"c4ai-command-a-03-2025","modelId":"c4ai-command-a-03-2025","modelSlug":"c4ai-command-a-03-2025","name":"Command A","organizationSlug":"cohere","n":29,"rankEligible":true,"score":123.99,"scoreLo":120.57,"scoreHi":127.03,"modelScoreStateId":"c4ai-command-a-03-2025::default","modelScoreConfigSlug":"c4ai-command-a-03-2025","modelScoreStateName":"Command A","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":448.4797,"rankLo":417.0,"rankHi":483.0},{"slug":"qwen2-5-32b-instruct","modelId":"qwen2-5-32b-instruct","modelSlug":"qwen2-5-32b-instruct","name":"Qwen2.5 Instruct 32B","organizationSlug":"alibaba","n":38,"rankEligible":true,"score":123.95,"scoreLo":121.18,"scoreHi":126.56,"modelScoreStateId":"qwen2-5-32b-instruct::default","modelScoreConfigSlug":"qwen2-5-32b-instruct","modelScoreStateName":"Qwen2.5 Instruct 32B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":448.8733,"rankLo":425.0,"rankHi":473.0},{"slug":"deepseek-v3-base","modelId":"deepseek-v3-base","modelSlug":"deepseek-v3-base","name":"DeepSeek-V3-Base","organizationSlug":"deepseek","n":30,"rankEligible":true,"score":123.94,"scoreLo":120.41,"scoreHi":127.3,"modelScoreStateId":"deepseek-v3-base::default","modelScoreConfigSlug":"deepseek-v3-base","modelScoreStateName":"DeepSeek-V3-Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":449.2222,"rankLo":415.0,"rankHi":484.0},{"slug":"gemma-3-27b-it","modelId":"gemma-3-27b-it","modelSlug":"gemma-3-27b-it","name":"Gemma 3 27B IT","organizationSlug":"google","n":92,"rankEligible":true,"score":123.93,"scoreLo":121.32,"scoreHi":126.26,"modelScoreStateId":"gemma-3-27b-it::default","modelScoreConfigSlug":"gemma-3-27b-it","modelScoreStateName":"Gemma 3 27B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":449.0842,"rankLo":431.0,"rankHi":468.0},{"slug":"mistral-large-instruct-2407","modelId":"mistral-large-instruct-2407","modelSlug":"mistral-large-instruct-2407","name":"Mistral Large 2 (Instruct 2407)","organizationSlug":"mistral-ai","n":43,"rankEligible":true,"score":123.85,"scoreLo":120.96,"scoreHi":126.48,"modelScoreStateId":"mistral-large-instruct-2407::default","modelScoreConfigSlug":"mistral-large-instruct-2407","modelScoreStateName":"Mistral Large 2 (Instruct 2407)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":450.2039,"rankLo":425.0,"rankHi":476.0},{"slug":"nemotron-4-340b-base","modelId":"nemotron-4-340b-base","modelSlug":"nemotron-4-340b-base","name":"Nemotron-4 340B Base","organizationSlug":"nvidia","n":6,"rankEligible":true,"score":123.84,"scoreLo":117.61,"scoreHi":130.28,"modelScoreStateId":"nemotron-4-340b-base::default","modelScoreConfigSlug":"nemotron-4-340b-base","modelScoreStateName":"Nemotron-4 340B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":451.8347,"rankLo":373.0,"rankHi":528.0},{"slug":"internvl3-8b","modelId":"internvl3-8b","modelSlug":"internvl3-8b","name":"InternVL3-8B","organizationSlug":"opengvlab","n":7,"rankEligible":true,"score":123.75,"scoreLo":118.45,"scoreHi":128.66,"modelScoreStateId":"internvl3-8b::default","modelScoreConfigSlug":"internvl3-8b","modelScoreStateName":"InternVL3-8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":452.6347,"rankLo":397.0,"rankHi":515.0},{"slug":"claude-3-opus-20240229","modelId":"claude-3-opus-20240229","modelSlug":"claude-3-opus-20240229","name":"Opus 3","organizationSlug":"anthropic","n":66,"rankEligible":true,"score":123.66,"scoreLo":120.98,"scoreHi":126.09,"modelScoreStateId":"claude-3-opus-20240229::default","modelScoreConfigSlug":"claude-3-opus-20240229","modelScoreStateName":"Opus 3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":452.5603,"rankLo":432.0,"rankHi":474.0},{"slug":"devstral-small-2507","modelId":"devstral-small-2507","modelSlug":"devstral-small-2507","name":"Devstral Small 1.1","organizationSlug":"mistral-ai","n":11,"rankEligible":true,"score":123.64,"scoreLo":118.43,"scoreHi":128.2,"modelScoreStateId":"devstral-small-2507::default","modelScoreConfigSlug":"devstral-small-2507","modelScoreStateName":"Devstral Small 1.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":453.6719,"rankLo":403.0,"rankHi":515.0},{"slug":"lfm2-5-8b-a1b","modelId":"lfm2-5-8b-a1b","modelSlug":"lfm2-5-8b-a1b","name":"LFM2.5-8B-A1B","organizationSlug":"liquid-ai","n":6,"rankEligible":true,"score":123.64,"scoreLo":113.26,"scoreHi":132.32,"modelScoreStateId":"lfm2-5-8b-a1b::default","modelScoreConfigSlug":"lfm2-5-8b-a1b","modelScoreStateName":"LFM2.5-8B-A1B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":454.2406,"rankLo":345.975,"rankHi":578.0},{"slug":"sarvam-m","modelId":"sarvam-m","modelSlug":"sarvam-m","name":"Sarvam M","organizationSlug":"sarvam-ai","n":7,"rankEligible":true,"score":123.57,"scoreLo":117.46,"scoreHi":129.08,"modelScoreStateId":"sarvam-m::enabled","modelScoreConfigSlug":"sarvam-m-thinking","modelScoreStateName":"Sarvam M (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":455.0128,"rankLo":393.0,"rankHi":525.025},{"slug":"nova-pro-v1-0","modelId":"nova-pro-v1-0","modelSlug":"nova-pro-v1-0","name":"Nova Pro","organizationSlug":"amazon","n":33,"rankEligible":true,"score":123.51,"scoreLo":120.32,"scoreHi":126.53,"modelScoreStateId":"nova-pro-v1-0::default","modelScoreConfigSlug":"nova-pro-v1-0","modelScoreStateName":"Nova Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":454.8158,"rankLo":426.0,"rankHi":488.0},{"slug":"lfm2-5-2-6b","modelId":"lfm2-5-2-6b","modelSlug":"lfm2-5-2-6b","name":"LFM2.5-2.6B","organizationSlug":"liquid-ai","n":15,"rankEligible":true,"score":123.41,"scoreLo":118.98,"scoreHi":127.41,"modelScoreStateId":"lfm2-5-2-6b::default","modelScoreConfigSlug":"lfm2-5-2-6b","modelScoreStateName":"LFM2.5-2.6B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":456.4819,"rankLo":412.0,"rankHi":510.0},{"slug":"qwen2-5-14b-instruct-1m","modelId":"qwen2-5-14b-instruct-1m","modelSlug":"qwen2-5-14b-instruct-1m","name":"Qwen2.5 14B Instruct 1M","organizationSlug":"alibaba","n":10,"rankEligible":true,"score":123.41,"scoreLo":118.9,"scoreHi":127.84,"modelScoreStateId":"qwen2-5-14b-instruct-1m::default","modelScoreConfigSlug":"qwen2-5-14b-instruct-1m","modelScoreStateName":"Qwen2.5 14B Instruct 1M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":456.6053,"rankLo":406.0,"rankHi":509.0},{"slug":"gemini-1-5-flash-002","modelId":"gemini-1-5-flash-002","modelSlug":"gemini-1-5-flash-002","name":"Gemini 1.5 Flash 002","organizationSlug":"google","n":42,"rankEligible":true,"score":123.39,"scoreLo":120.24,"scoreHi":126.2,"modelScoreStateId":"gemini-1-5-flash-002::default","modelScoreConfigSlug":"gemini-1-5-flash-002","modelScoreStateName":"Gemini 1.5 Flash 002","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":456.2678,"rankLo":429.0,"rankHi":486.025},{"slug":"ling-mini-2-0","modelId":"ling-mini-2-0","modelSlug":"ling-mini-2-0","name":"Ling Mini 2.0","organizationSlug":"inclusion-ai","n":6,"rankEligible":true,"score":123.36,"scoreLo":117.47,"scoreHi":128.58,"modelScoreStateId":"ling-mini-2-0::default","modelScoreConfigSlug":"ling-mini-2-0","modelScoreStateName":"Ling Mini 2.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":457.6194,"rankLo":397.0,"rankHi":526.0},{"slug":"deepseek-coder-v2-instruct","modelId":"deepseek-coder-v2-instruct","modelSlug":"deepseek-coder-v2-instruct","name":"DeepSeek-Coder-V2-Instruct","organizationSlug":"deepseek","n":25,"rankEligible":true,"score":122.81,"scoreLo":119.39,"scoreHi":126.01,"modelScoreStateId":"deepseek-coder-v2-instruct::default","modelScoreConfigSlug":"deepseek-coder-v2-instruct","modelScoreStateName":"DeepSeek-Coder-V2-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":464.0358,"rankLo":433.0,"rankHi":499.0},{"slug":"olmo-3-7b-think","modelId":"olmo-3-7b-think","modelSlug":"olmo-3-7b-think","name":"Olmo 3 7B Think","organizationSlug":"allen-ai","n":10,"rankEligible":true,"score":122.77,"scoreLo":117.1,"scoreHi":127.93,"modelScoreStateId":"olmo-3-7b-think::default","modelScoreConfigSlug":"olmo-3-7b-think","modelScoreStateName":"Olmo 3 7B Think","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":465.2919,"rankLo":405.0,"rankHi":532.025},{"slug":"gemini-1-5-pro-001","modelId":"gemini-1-5-pro-001","modelSlug":"gemini-1-5-pro-001","name":"Gemini 1.5 Pro 001","organizationSlug":"google","n":8,"rankEligible":true,"score":122.72,"scoreLo":118.02,"scoreHi":126.98,"modelScoreStateId":"gemini-1-5-pro-001::default","modelScoreConfigSlug":"gemini-1-5-pro-001","modelScoreStateName":"Gemini 1.5 Pro 001","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":465.7703,"rankLo":417.0,"rankHi":519.0},{"slug":"palmyra-x-004","modelId":"palmyra-x-004","modelSlug":"palmyra-x-004","name":"Palmyra X 004","organizationSlug":"writer","n":6,"rankEligible":true,"score":122.61,"scoreLo":116.93,"scoreHi":128.09,"modelScoreStateId":"palmyra-x-004::default","modelScoreConfigSlug":"palmyra-x-004","modelScoreStateName":"Palmyra X 004","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":467.4997,"rankLo":402.0,"rankHi":534.0},{"slug":"llama-4-maverick-17b-128e","modelId":"llama-4-maverick-17b-128e","modelSlug":"llama-4-maverick-17b-128e","name":"Llama 4 Maverick","organizationSlug":"meta","n":17,"rankEligible":true,"score":122.56,"scoreLo":118.12,"scoreHi":126.68,"modelScoreStateId":"llama-4-maverick-17b-128e::default","modelScoreConfigSlug":"llama-4-maverick-17b-128e","modelScoreStateName":"Llama 4 Maverick","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":468.005,"rankLo":420.975,"rankHi":518.0},{"slug":"deepseek-v2-5","modelId":"deepseek-v2-5","modelSlug":"deepseek-v2-5","name":"DeepSeek-V2.5","organizationSlug":"deepseek","n":35,"rankEligible":true,"score":122.44,"scoreLo":119.32,"scoreHi":125.12,"modelScoreStateId":"deepseek-v2-5::default","modelScoreConfigSlug":"deepseek-v2-5","modelScoreStateName":"DeepSeek-V2.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":469.1906,"rankLo":443.0,"rankHi":497.0},{"slug":"deepseek-r1-distill-qwen-14b","modelId":"deepseek-r1-distill-qwen-14b","modelSlug":"deepseek-r1-distill-qwen-14b","name":"DeepSeek R1 Distill Qwen 14B","organizationSlug":"deepseek","n":31,"rankEligible":true,"score":122.39,"scoreLo":118.94,"scoreHi":125.47,"modelScoreStateId":"deepseek-r1-distill-qwen-14b::default","modelScoreConfigSlug":"deepseek-r1-distill-qwen-14b","modelScoreStateName":"DeepSeek R1 Distill Qwen 14B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":470.0439,"rankLo":438.0,"rankHi":504.0},{"slug":"qwen2-5-32b","modelId":"qwen2-5-32b","modelSlug":"qwen2-5-32b","name":"Qwen2.5 32B","organizationSlug":"alibaba","n":25,"rankEligible":true,"score":122.28,"scoreLo":118.87,"scoreHi":125.28,"modelScoreStateId":"qwen2-5-32b::default","modelScoreConfigSlug":"qwen2-5-32b","modelScoreStateName":"Qwen2.5 32B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":471.4397,"rankLo":441.0,"rankHi":504.0},{"slug":"nanbeige4-3b-thinking-2511","modelId":"nanbeige4-3b-thinking-2511","modelSlug":"nanbeige4-3b-thinking-2511","name":"Nanbeige4-3B-Thinking-2511","organizationSlug":"nanbeige","n":4,"rankEligible":true,"score":122.24,"scoreLo":113.36,"scoreHi":130.51,"modelScoreStateId":"nanbeige4-3b-thinking-2511::default","modelScoreConfigSlug":"nanbeige4-3b-thinking-2511","modelScoreStateName":"Nanbeige4-3B-Thinking-2511","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":472.3047,"rankLo":372.975,"rankHi":572.0},{"slug":"qwen3-5-2b","modelId":"qwen3-5-2b","modelSlug":"qwen3-5-2b","name":"Qwen3.5-2B","organizationSlug":"alibaba","n":74,"rankEligible":true,"score":121.96,"scoreLo":118.92,"scoreHi":124.77,"modelScoreStateId":"qwen3-5-2b::enabled","modelScoreConfigSlug":"qwen3-5-2b-thinking","modelScoreStateName":"Qwen3.5-2B (thinking)","modelScoreStateBestProbability":0.9994444444444445,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":475.7458,"rankLo":450.0,"rankHi":502.0},{"slug":"deepseek-coder-v2-instruct-0724","modelId":"deepseek-coder-v2-instruct-0724","modelSlug":"deepseek-coder-v2-instruct-0724","name":"DeepSeek-Coder-V2-Instruct-0724","organizationSlug":"deepseek","n":10,"rankEligible":true,"score":121.95,"scoreLo":118.08,"scoreHi":125.32,"modelScoreStateId":"deepseek-coder-v2-instruct-0724::default","modelScoreConfigSlug":"deepseek-coder-v2-instruct-0724","modelScoreStateName":"DeepSeek-Coder-V2-Instruct-0724","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":476.0989,"rankLo":438.0,"rankHi":518.0},{"slug":"llama-3-2-90b-vision-instruct","modelId":"llama-3-2-90b-vision-instruct","modelSlug":"llama-3-2-90b-vision-instruct","name":"Llama 3.2 90B Vision Instruct","organizationSlug":"meta","n":26,"rankEligible":true,"score":121.93,"scoreLo":118.38,"scoreHi":125.11,"modelScoreStateId":"llama-3-2-90b-vision-instruct::default","modelScoreConfigSlug":"llama-3-2-90b-vision-instruct","modelScoreStateName":"Llama 3.2 90B Vision Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":476.1786,"rankLo":441.975,"rankHi":511.0},{"slug":"gemini-1-5-pro-preview-0514","modelId":"gemini-1-5-pro-preview-0514","modelSlug":"gemini-1-5-pro-preview-0514","name":"Gemini 1.5 Pro Preview 0514","organizationSlug":"google","n":7,"rankEligible":true,"score":121.88,"scoreLo":117.43,"scoreHi":126.37,"modelScoreStateId":"gemini-1-5-pro-preview-0514::default","modelScoreConfigSlug":"gemini-1-5-pro-api-0514-benchmark-entry","modelScoreStateName":"Gemini 1.5 Pro Preview 0514","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":477.2342,"rankLo":424.0,"rankHi":526.0},{"slug":"llama-3-1-70b-instruct","modelId":"llama-3-1-70b-instruct","modelSlug":"llama-3-1-70b-instruct","name":"Llama 3.1 70B Instruct","organizationSlug":"meta","n":70,"rankEligible":true,"score":121.8,"scoreLo":118.81,"scoreHi":124.36,"modelScoreStateId":"llama-3-1-70b-instruct::default","modelScoreConfigSlug":"llama-3-1-70b-instruct","modelScoreStateName":"Llama 3.1 70B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":478.1181,"rankLo":455.0,"rankHi":502.0},{"slug":"qwen2-5-72b","modelId":"qwen2-5-72b","modelSlug":"qwen2-5-72b","name":"Qwen2.5 72B","organizationSlug":"alibaba","n":39,"rankEligible":true,"score":121.75,"scoreLo":118.24,"scoreHi":124.98,"modelScoreStateId":"qwen2-5-72b::default","modelScoreConfigSlug":"qwen2-5-72b","modelScoreStateName":"Qwen2.5 72B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":478.7617,"rankLo":445.0,"rankHi":511.0},{"slug":"phi-4","modelId":"phi-4","modelSlug":"phi-4","name":"Phi-4","organizationSlug":"microsoft","n":44,"rankEligible":true,"score":121.73,"scoreLo":118.38,"scoreHi":124.66,"modelScoreStateId":"phi-4::default","modelScoreConfigSlug":"phi-4","modelScoreStateName":"Phi-4","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":479.1803,"rankLo":451.0,"rankHi":509.0},{"slug":"nemotron-h-56b-base-8k","modelId":"nemotron-h-56b-base-8k","modelSlug":"nemotron-h-56b-base-8k","name":"Nemotron-H 56B Base 8K","organizationSlug":"nvidia","n":12,"rankEligible":true,"score":121.72,"scoreLo":116.44,"scoreHi":126.79,"modelScoreStateId":"nemotron-h-56b-base-8k::default","modelScoreConfigSlug":"nemotron-h-56b-base-8k","modelScoreStateName":"Nemotron-H 56B Base 8K","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":479.2358,"rankLo":419.0,"rankHi":536.025},{"slug":"qwen-turbo-2024-11-01","modelId":"qwen-turbo-2024-11-01","modelSlug":"qwen-turbo-2024-11-01","name":"Qwen2.5-Turbo","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":121.72,"scoreLo":117.62,"scoreHi":125.64,"modelScoreStateId":"qwen-turbo-2024-11-01::default","modelScoreConfigSlug":"qwen-turbo-2024-11-01","modelScoreStateName":"Qwen2.5-Turbo","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":479.2347,"rankLo":435.0,"rankHi":520.0},{"slug":"qwen2-72b","modelId":"qwen2-72b","modelSlug":"qwen2-72b","name":"Qwen2-72B","organizationSlug":"alibaba","n":27,"rankEligible":true,"score":121.67,"scoreLo":118.04,"scoreHi":124.91,"modelScoreStateId":"qwen2-72b::default","modelScoreConfigSlug":"qwen2-72b","modelScoreStateName":"Qwen2-72B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":479.9922,"rankLo":445.0,"rankHi":515.0},{"slug":"llama-3-1-nemotron-70b-instruct-hf","modelId":"llama-3-1-nemotron-70b-instruct-hf","modelSlug":"llama-3-1-nemotron-70b-instruct-hf","name":"Llama 3.1 Nemotron 70B Instruct HF","organizationSlug":"nvidia","n":24,"rankEligible":true,"score":121.6,"scoreLo":117.99,"scoreHi":124.99,"modelScoreStateId":"llama-3-1-nemotron-70b-instruct-hf::default","modelScoreConfigSlug":"llama-3-1-nemotron-70b-instruct-hf","modelScoreStateName":"Llama 3.1 Nemotron 70B Instruct HF","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":481.0103,"rankLo":445.0,"rankHi":515.0},{"slug":"gpt-4o-mini","modelId":"gpt-4o-mini","modelSlug":"gpt-4o-mini","name":"GPT-4o Mini","organizationSlug":"openai","n":106,"rankEligible":true,"score":121.49,"scoreLo":118.71,"scoreHi":123.95,"modelScoreStateId":"gpt-4o-mini::default","modelScoreConfigSlug":"gpt-4o-mini","modelScoreStateName":"GPT-4o Mini","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":482.5356,"rankLo":462.0,"rankHi":503.0},{"slug":"qwen3-30b-a3b-base","modelId":"qwen3-30b-a3b-base","modelSlug":"qwen3-30b-a3b-base","name":"Qwen3-30B-A3B-Base","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":121.48,"scoreLo":117.54,"scoreHi":125.11,"modelScoreStateId":"qwen3-30b-a3b-base::default","modelScoreConfigSlug":"qwen3-30b-a3b-base","modelScoreStateName":"Qwen3-30B-A3B-Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":482.6314,"rankLo":441.0,"rankHi":522.0},{"slug":"gpt-4-0613","modelId":"gpt-4-0613","modelSlug":"gpt-4-0613","name":"GPT-4 0613","organizationSlug":"openai","n":17,"rankEligible":true,"score":121.44,"scoreLo":117.28,"scoreHi":125.26,"modelScoreStateId":"gpt-4-0613::default","modelScoreConfigSlug":"gpt-4-0613","modelScoreStateName":"GPT-4 0613","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":483.0714,"rankLo":441.0,"rankHi":525.0},{"slug":"olmo-3-1-32b-instruct","modelId":"olmo-3-1-32b-instruct","modelSlug":"olmo-3-1-32b-instruct","name":"Olmo 3.1 32B Instruct","organizationSlug":"allen-ai","n":8,"rankEligible":true,"score":121.35,"scoreLo":113.54,"scoreHi":128.58,"modelScoreStateId":"olmo-3-1-32b-instruct::default","modelScoreConfigSlug":"olmo-3-1-32b-instruct","modelScoreStateName":"Olmo 3.1 32B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":483.7072,"rankLo":397.975,"rankHi":572.0},{"slug":"mistral-small-24b-instruct-2501","modelId":"mistral-small-24b-instruct-2501","modelSlug":"mistral-small-24b-instruct-2501","name":"Mistral Small 3 24B Instruct 2501","organizationSlug":"mistral-ai","n":23,"rankEligible":true,"score":121.15,"scoreLo":117.17,"scoreHi":124.58,"modelScoreStateId":"mistral-small-24b-instruct-2501::default","modelScoreConfigSlug":"mistral-small-24b-instruct-2501","modelScoreStateName":"Mistral Small 3 24B Instruct 2501","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":487.2133,"rankLo":446.0,"rankHi":525.0},{"slug":"solar-pro-preview-instruct","modelId":"solar-pro-preview-instruct","modelSlug":"solar-pro-preview-instruct","name":"Solar Pro Instruct Preview","organizationSlug":"upstage","n":8,"rankEligible":true,"score":121.08,"scoreLo":116.61,"scoreHi":125.28,"modelScoreStateId":"solar-pro-preview-instruct::default","modelScoreConfigSlug":"solar-pro-preview-instruct","modelScoreStateName":"Solar Pro Instruct Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":487.885,"rankLo":440.0,"rankHi":533.025},{"slug":"nemotron-h-47b-base-8k","modelId":"nemotron-h-47b-base-8k","modelSlug":"nemotron-h-47b-base-8k","name":"Nemotron-H 47B Base 8K","organizationSlug":"nvidia","n":12,"rankEligible":true,"score":121.06,"scoreLo":115.66,"scoreHi":126.19,"modelScoreStateId":"nemotron-h-47b-base-8k::default","modelScoreConfigSlug":"nemotron-h-47b-base-8k","modelScoreStateName":"Nemotron-H 47B Base 8K","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":488.0878,"rankLo":426.0,"rankHi":544.0},{"slug":"qwen2-72b-instruct","modelId":"qwen2-72b-instruct","modelSlug":"qwen2-72b-instruct","name":"Qwen2 72B Instruct","organizationSlug":"alibaba","n":35,"rankEligible":true,"score":121.06,"scoreLo":117.84,"scoreHi":123.85,"modelScoreStateId":"qwen2-72b-instruct::default","modelScoreConfigSlug":"qwen2-72b-instruct","modelScoreStateName":"Qwen2 72B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":488.7644,"rankLo":462.0,"rankHi":516.0},{"slug":"qwen2-5-coder-32b-instruct","modelId":"qwen2-5-coder-32b-instruct","modelSlug":"qwen2-5-coder-32b-instruct","name":"Qwen2.5 Coder 32B Instruct","organizationSlug":"alibaba","n":25,"rankEligible":true,"score":121.03,"scoreLo":117.3,"scoreHi":124.64,"modelScoreStateId":"qwen2-5-coder-32b-instruct::default","modelScoreConfigSlug":"qwen2-5-coder-32b-instruct","modelScoreStateName":"Qwen2.5 Coder 32B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":488.7428,"rankLo":449.0,"rankHi":526.0},{"slug":"gemma-4-e2b-it","modelId":"gemma-4-e2b-it","modelSlug":"gemma-4-e2b-it","name":"Gemma 4 E2B IT","organizationSlug":"google","n":69,"rankEligible":true,"score":120.92,"scoreLo":117.16,"scoreHi":124.11,"modelScoreStateId":"gemma-4-e2b-it::enabled","modelScoreConfigSlug":"gemma-4-e2b-it-thinking","modelScoreStateName":"Gemma 4 E2B IT (thinking)","modelScoreStateBestProbability":0.7422222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":490.3681,"rankLo":457.0,"rankHi":523.0},{"slug":"nemotron-3-nano-30b-a3b-base-bf16","modelId":"nemotron-3-nano-30b-a3b-base-bf16","modelSlug":"nemotron-3-nano-30b-a3b-base-bf16","name":"Nemotron 3 Nano 30B A3B Base","organizationSlug":"nvidia","n":10,"rankEligible":true,"score":120.87,"scoreLo":116.23,"scoreHi":125.24,"modelScoreStateId":"nemotron-3-nano-30b-a3b-base-bf16::default","modelScoreConfigSlug":"nemotron-3-nano-30b-a3b-base-bf16","modelScoreStateName":"Nemotron 3 Nano 30B A3B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":490.6231,"rankLo":438.0,"rankHi":537.0},{"slug":"gemini-ultra","modelId":"gemini-ultra","modelSlug":"gemini-ultra","name":"Gemini 1.0 Ultra","organizationSlug":"google","n":12,"rankEligible":true,"score":120.72,"scoreLo":116.03,"scoreHi":125.06,"modelScoreStateId":"gemini-ultra::default","modelScoreConfigSlug":"gemini-ultra","modelScoreStateName":"Gemini 1.0 Ultra","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":492.7144,"rankLo":439.0,"rankHi":541.0},{"slug":"kimi-vl-a3b-instruct","modelId":"kimi-vl-a3b-instruct","modelSlug":"kimi-vl-a3b-instruct","name":"Kimi-VL-A3B-Instruct","organizationSlug":"moonshot-ai","n":19,"rankEligible":true,"score":120.72,"scoreLo":116.47,"scoreHi":124.4,"modelScoreStateId":"kimi-vl-a3b-instruct::default","modelScoreConfigSlug":"kimi-vl-a3b-instruct","modelScoreStateName":"Kimi-VL-A3B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":492.8236,"rankLo":450.0,"rankHi":531.0},{"slug":"deepseek-v2-chat-0628","modelId":"deepseek-v2-chat-0628","modelSlug":"deepseek-v2-chat-0628","name":"DeepSeek-V2-Chat-0628","organizationSlug":"deepseek","n":15,"rankEligible":true,"score":120.71,"scoreLo":117.09,"scoreHi":124.02,"modelScoreStateId":"deepseek-v2-chat-0628::default","modelScoreConfigSlug":"deepseek-v2-chat-0628","modelScoreStateName":"DeepSeek-V2-Chat-0628","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":493.3475,"rankLo":457.975,"rankHi":526.0},{"slug":"codestral-2501","modelId":"codestral-2501","modelSlug":"codestral-2501","name":"Codestral 25.01","organizationSlug":"mistral-ai","n":9,"rankEligible":true,"score":120.7,"scoreLo":114.82,"scoreHi":125.98,"modelScoreStateId":"codestral-2501::default","modelScoreConfigSlug":"codestral-2501","modelScoreStateName":"Codestral 25.01","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":492.3436,"rankLo":427.0,"rankHi":555.0},{"slug":"qwen3-14b-base","modelId":"qwen3-14b-base","modelSlug":"qwen3-14b-base","name":"Qwen3 14B Base","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":120.65,"scoreLo":116.49,"scoreHi":124.39,"modelScoreStateId":"qwen3-14b-base::default","modelScoreConfigSlug":"qwen3-14b-base","modelScoreStateName":"Qwen3 14B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":493.8247,"rankLo":451.0,"rankHi":533.0},{"slug":"qwen2-5-vl-7b-instruct","modelId":"qwen2-5-vl-7b-instruct","modelSlug":"qwen2-5-vl-7b-instruct","name":"Qwen2.5-VL-7B-Instruct","organizationSlug":"alibaba","n":35,"rankEligible":true,"score":120.63,"scoreLo":116.93,"scoreHi":124.03,"modelScoreStateId":"qwen2-5-vl-7b-instruct::default","modelScoreConfigSlug":"qwen2-5-vl-7b-instruct","modelScoreStateName":"Qwen2.5-VL-7B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":494.3619,"rankLo":460.0,"rankHi":525.0},{"slug":"kimi-vl-a3b-thinking","modelId":"kimi-vl-a3b-thinking","modelSlug":"kimi-vl-a3b-thinking","name":"Kimi-VL-A3B-Thinking","organizationSlug":"moonshot-ai","n":11,"rankEligible":true,"score":120.61,"scoreLo":116.21,"scoreHi":124.65,"modelScoreStateId":"kimi-vl-a3b-thinking::default","modelScoreConfigSlug":"kimi-vl-a3b-thinking-thinking","modelScoreStateName":"Kimi-VL-A3B-Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":494.3364,"rankLo":448.0,"rankHi":538.0},{"slug":"qwen2-5-14b-instruct","modelId":"qwen2-5-14b-instruct","modelSlug":"qwen2-5-14b-instruct","name":"Qwen2.5 14B Instruct","organizationSlug":"alibaba","n":31,"rankEligible":true,"score":120.54,"scoreLo":117.02,"scoreHi":123.57,"modelScoreStateId":"qwen2-5-14b-instruct::default","modelScoreConfigSlug":"qwen2-5-14b-instruct","modelScoreStateName":"Qwen2.5 14B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":495.8608,"rankLo":466.0,"rankHi":523.0},{"slug":"qwen3-vl-2b-thinking","modelId":"qwen3-vl-2b-thinking","modelSlug":"qwen3-vl-2b-thinking","name":"Qwen3 VL 2B Thinking","organizationSlug":"alibaba","n":43,"rankEligible":true,"score":120.43,"scoreLo":117.11,"scoreHi":123.37,"modelScoreStateId":"qwen3-vl-2b-thinking::default","modelScoreConfigSlug":"qwen3-vl-2b-thinking","modelScoreStateName":"Qwen3 VL 2B Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":497.3758,"rankLo":470.0,"rankHi":522.0},{"slug":"llama-3-1-405b","modelId":"llama-3-1-405b","modelSlug":"llama-3-1-405b","name":"Llama 3.1 405B","organizationSlug":"meta","n":27,"rankEligible":true,"score":120.33,"scoreLo":116.35,"scoreHi":123.88,"modelScoreStateId":"llama-3-1-405b::default","modelScoreConfigSlug":"llama-3-1-405b","modelScoreStateName":"Llama 3.1 405B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":498.2264,"rankLo":459.0,"rankHi":532.0},{"slug":"exaone-4-0-1-2b","modelId":"exaone-4-0-1-2b","modelSlug":"exaone-4-0-1-2b","name":"EXAONE 4.0 1.2B","organizationSlug":"lg-ai-research","n":7,"rankEligible":true,"score":120.26,"scoreLo":114.45,"scoreHi":125.35,"modelScoreStateId":"exaone-4-0-1-2b::enabled","modelScoreConfigSlug":"exaone-4-0-1-2b-thinking","modelScoreStateName":"EXAONE 4.0 1.2B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":498.1214,"rankLo":435.0,"rankHi":557.0},{"slug":"sarvam-105b","modelId":"sarvam-105b","modelSlug":"sarvam-105b","name":"Sarvam 105B","organizationSlug":"sarvam-ai","n":5,"rankEligible":true,"score":120.23,"scoreLo":108.05,"scoreHi":129.82,"modelScoreStateId":"sarvam-105b::effort:high","modelScoreConfigSlug":"sarvam-105b-high","modelScoreStateName":"Sarvam 105B (high)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"effort:high","modelScoreReasoningEffort":"high","modelScoreHarness":"none","rankMean":497.1597,"rankLo":381.975,"rankHi":636.025},{"slug":"granite-4-0-h-small","modelId":"granite-4-0-h-small","modelSlug":"granite-4-0-h-small","name":"Granite 4.0 H Small","organizationSlug":"ibm","n":13,"rankEligible":true,"score":120.18,"scoreLo":115.12,"scoreHi":124.49,"modelScoreStateId":"granite-4-0-h-small::default","modelScoreConfigSlug":"granite-4-0-h-small","modelScoreStateName":"Granite 4.0 H Small","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":499.6189,"rankLo":447.975,"rankHi":550.0},{"slug":"llama-3-1-tulu-3-70b-dpo","modelId":"llama-3-1-tulu-3-70b-dpo","modelSlug":"llama-3-1-tulu-3-70b-dpo","name":"Llama 3.1 T\u00fclu 3 70B DPO","organizationSlug":"allen-ai","n":7,"rankEligible":true,"score":120.16,"scoreLo":115.04,"scoreHi":124.78,"modelScoreStateId":"llama-3-1-tulu-3-70b-dpo::default","modelScoreConfigSlug":"llama-3-1-tulu-3-70b-dpo","modelScoreStateName":"Llama 3.1 T\u00fclu 3 70B DPO","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":499.7558,"rankLo":447.0,"rankHi":551.0},{"slug":"nvlm-d-72b","modelId":"nvlm-d-72b","modelSlug":"nvlm-d-72b","name":"NVLM-D-72B","organizationSlug":"nvidia","n":18,"rankEligible":true,"score":120.08,"scoreLo":115.73,"scoreHi":124.05,"modelScoreStateId":"nvlm-d-72b::default","modelScoreConfigSlug":"nvlm-d-72b","modelScoreStateName":"NVLM-D-72B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":501.1183,"rankLo":454.0,"rankHi":541.0},{"slug":"nemotron-nano-12b-v2-base","modelId":"nemotron-nano-12b-v2-base","modelSlug":"nemotron-nano-12b-v2-base","name":"Nemotron Nano 12B V2 Base","organizationSlug":"nvidia","n":12,"rankEligible":true,"score":120.0,"scoreLo":115.4,"scoreHi":124.48,"modelScoreStateId":"nemotron-nano-12b-v2-base::default","modelScoreConfigSlug":"nemotron-nano-12b-v2-base","modelScoreStateName":"Nemotron Nano 12B V2 Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":501.9608,"rankLo":450.0,"rankHi":545.0},{"slug":"nemotron-3-5-lightning-30b-a3b-base-bf16","modelId":"nemotron-3-5-lightning-30b-a3b-base-bf16","modelSlug":"nemotron-3-5-lightning-30b-a3b-base-bf16","name":"Nemotron 3.5 Lightning 30B A3B Base","organizationSlug":"nvidia","n":11,"rankEligible":true,"score":119.92,"scoreLo":115.52,"scoreHi":123.82,"modelScoreStateId":"nemotron-3-5-lightning-30b-a3b-base-bf16::default","modelScoreConfigSlug":"nemotron-3-5-lightning-30b-a3b-base-bf16","modelScoreStateName":"Nemotron 3.5 Lightning 30B A3B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":503.3431,"rankLo":460.975,"rankHi":543.0},{"slug":"jamba-large-1-7","modelId":"jamba-large-1-7","modelSlug":"jamba-large-1-7","name":"Jamba Large 1.7","organizationSlug":"ai21-labs","n":10,"rankEligible":true,"score":119.86,"scoreLo":114.18,"scoreHi":125.08,"modelScoreStateId":"jamba-large-1-7::default","modelScoreConfigSlug":"jamba-large-1-7","modelScoreStateName":"Jamba Large 1.7","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":503.3861,"rankLo":441.0,"rankHi":561.0},{"slug":"gemini-1-5-flash-preview-0514","modelId":"gemini-1-5-flash-preview-0514","modelSlug":"gemini-1-5-flash-preview-0514","name":"Gemini 1.5 Flash Preview 0514","organizationSlug":"google","n":9,"rankEligible":true,"score":119.68,"scoreLo":114.76,"scoreHi":124.38,"modelScoreStateId":"gemini-1-5-flash-preview-0514::default","modelScoreConfigSlug":"gemini-1-5-flash-api-0514-benchmark-entry","modelScoreStateName":"Gemini 1.5 Flash Preview 0514","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":506.2206,"rankLo":448.0,"rankHi":553.0},{"slug":"gpt-4-1-nano","modelId":"gpt-4-1-nano","modelSlug":"gpt-4-1-nano","name":"GPT-4.1 Nano","organizationSlug":"openai","n":81,"rankEligible":true,"score":119.63,"scoreLo":116.42,"scoreHi":122.4,"modelScoreStateId":"gpt-4-1-nano::default","modelScoreConfigSlug":"gpt-4-1-nano","modelScoreStateName":"GPT-4.1 Nano","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":508.27,"rankLo":488.0,"rankHi":527.0},{"slug":"gemini-1-5-pro-preview-0409","modelId":"gemini-1-5-pro-preview-0409","modelSlug":"gemini-1-5-pro-preview-0409","name":"Gemini 1.5 Pro Preview 0409","organizationSlug":"google","n":4,"rankEligible":true,"score":119.61,"scoreLo":114.63,"scoreHi":124.62,"modelScoreStateId":"gemini-1-5-pro-preview-0409::default","modelScoreConfigSlug":"gemini-1-5-pro-preview-0409","modelScoreStateName":"Gemini 1.5 Pro Preview 0409","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":506.8861,"rankLo":444.975,"rankHi":554.0},{"slug":"gemma-3-12b-it","modelId":"gemma-3-12b-it","modelSlug":"gemma-3-12b-it","name":"Gemma 3 12B IT","organizationSlug":"google","n":75,"rankEligible":true,"score":119.6,"scoreLo":116.31,"scoreHi":122.43,"modelScoreStateId":"gemma-3-12b-it::default","modelScoreConfigSlug":"gemma-3-12b-it","modelScoreStateName":"Gemma 3 12B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":508.6161,"rankLo":487.0,"rankHi":528.025},{"slug":"gpt-4-0314","modelId":"gpt-4-0314","modelSlug":"gpt-4-0314","name":"GPT-4 (0314)","organizationSlug":"openai","n":11,"rankEligible":true,"score":119.57,"scoreLo":115.48,"scoreHi":123.16,"modelScoreStateId":"gpt-4-0314::default","modelScoreConfigSlug":"gpt-4-0314","modelScoreStateName":"GPT-4 (0314)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":508.1242,"rankLo":470.0,"rankHi":542.0},{"slug":"minicpm5-1b","modelId":"minicpm5-1b","modelSlug":"minicpm5-1b","name":"MiniCPM5-1B","organizationSlug":"openbmb","n":5,"rankEligible":true,"score":119.44,"scoreLo":111.33,"scoreHi":126.43,"modelScoreStateId":"minicpm5-1b::enabled","modelScoreConfigSlug":"minicpm5-1b-thinking","modelScoreStateName":"MiniCPM5-1B (thinking)","modelScoreStateBestProbability":0.9997222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":507.9011,"rankLo":421.0,"rankHi":600.0},{"slug":"qwen2-5-coder-14b-instruct","modelId":"qwen2-5-coder-14b-instruct","modelSlug":"qwen2-5-coder-14b-instruct","name":"Qwen2.5 Coder 14B Instruct","organizationSlug":"alibaba","n":14,"rankEligible":true,"score":119.17,"scoreLo":113.83,"scoreHi":124.15,"modelScoreStateId":"qwen2-5-coder-14b-instruct::default","modelScoreConfigSlug":"qwen2-5-coder-14b-instruct","modelScoreStateName":"Qwen2.5 Coder 14B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":512.1519,"rankLo":452.0,"rankHi":564.025},{"slug":"internvl2-40b","modelId":"internvl2-40b","modelSlug":"internvl2-40b","name":"InternVL2-40B","organizationSlug":"opengvlab","n":4,"rankEligible":true,"score":119.07,"scoreLo":110.95,"scoreHi":127.21,"modelScoreStateId":"internvl2-40b::default","modelScoreConfigSlug":"internvl2-40b","modelScoreStateName":"InternVL2-40B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":512.3017,"rankLo":412.0,"rankHi":605.025},{"slug":"qwq-32b-preview","modelId":"qwq-32b-preview","modelSlug":"qwq-32b-preview","name":"QwQ-32B-Preview","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":118.98,"scoreLo":113.59,"scoreHi":123.71,"modelScoreStateId":"qwq-32b-preview::default","modelScoreConfigSlug":"qwq-32b-preview","modelScoreStateName":"QwQ-32B-Preview","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":514.6694,"rankLo":460.0,"rankHi":564.0},{"slug":"yi-large","modelId":"yi-large","modelSlug":"yi-large","name":"Yi-Large","organizationSlug":"01-ai","n":4,"rankEligible":true,"score":118.72,"scoreLo":113.27,"scoreHi":123.96,"modelScoreStateId":"yi-large::default","modelScoreConfigSlug":"yi-large","modelScoreStateName":"Yi-Large","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":517.8942,"rankLo":455.0,"rankHi":571.0},{"slug":"nemotron-4-340b-instruct","modelId":"nemotron-4-340b-instruct","modelSlug":"nemotron-4-340b-instruct","name":"Nemotron-4 340B Instruct","organizationSlug":"nvidia","n":8,"rankEligible":true,"score":118.7,"scoreLo":113.52,"scoreHi":123.79,"modelScoreStateId":"nemotron-4-340b-instruct::default","modelScoreConfigSlug":"nemotron-4-340b-instruct","modelScoreStateName":"Nemotron-4 340B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":517.9236,"rankLo":457.0,"rankHi":569.0},{"slug":"ministral-3-3b-instruct-2512","modelId":"ministral-3-3b-instruct-2512","modelSlug":"ministral-3-3b-instruct-2512","name":"Ministral 3 3B Instruct 2512","organizationSlug":"mistral-ai","n":28,"rankEligible":true,"score":118.66,"scoreLo":114.61,"scoreHi":122.25,"modelScoreStateId":"ministral-3-3b-instruct-2512::default","modelScoreConfigSlug":"ministral-3-3b-instruct-2512","modelScoreStateName":"Ministral 3 3B Instruct 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":519.3089,"rankLo":484.0,"rankHi":552.0},{"slug":"llava-onevision-qwen2-72b-ov-sft","modelId":"llava-onevision-qwen2-72b-ov-sft","modelSlug":"llava-onevision-qwen2-72b-ov-sft","name":"LLaVA-OneVision 72B","organizationSlug":"llava-authors","n":7,"rankEligible":true,"score":118.65,"scoreLo":113.1,"scoreHi":123.58,"modelScoreStateId":"llava-onevision-qwen2-72b-ov-sft::default","modelScoreConfigSlug":"llava-onevision-qwen2-72b-ov-sft","modelScoreStateName":"LLaVA-OneVision 72B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":518.6339,"rankLo":458.0,"rankHi":573.0},{"slug":"llama-3-1-nemotron-nano-4b-v1-1","modelId":"llama-3-1-nemotron-nano-4b-v1-1","modelSlug":"llama-3-1-nemotron-nano-4b-v1-1","name":"Llama 3.1 Nemotron Nano 4B V1.1","organizationSlug":"nvidia","n":7,"rankEligible":true,"score":118.59,"scoreLo":112.43,"scoreHi":124.2,"modelScoreStateId":"llama-3-1-nemotron-nano-4b-v1-1::enabled","modelScoreConfigSlug":"llama-3-1-nemotron-nano-4b-v1-1-thinking","modelScoreStateName":"Llama 3.1 Nemotron Nano 4B V1.1 (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":519.2708,"rankLo":452.0,"rankHi":582.0},{"slug":"jamba-large-1-5","modelId":"jamba-large-1-5","modelSlug":"jamba-large-1-5","name":"Jamba 1.5 Large","organizationSlug":"ai21-labs","n":7,"rankEligible":true,"score":118.06,"scoreLo":112.7,"scoreHi":123.18,"modelScoreStateId":"jamba-large-1-5::default","modelScoreConfigSlug":"jamba-large-1-5","modelScoreStateName":"Jamba 1.5 Large","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":525.86,"rankLo":467.0,"rankHi":578.0},{"slug":"medgemma-1-5-4b-it","modelId":"medgemma-1-5-4b-it","modelSlug":"medgemma-1-5-4b-it","name":"MedGemma 1.5 4B IT","organizationSlug":"google","n":5,"rankEligible":true,"score":117.94,"scoreLo":106.95,"scoreHi":127.31,"modelScoreStateId":"medgemma-1-5-4b-it::default","modelScoreConfigSlug":"medgemma-1-5-4b-it","modelScoreStateName":"MedGemma 1.5 4B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":525.8183,"rankLo":411.0,"rankHi":650.025},{"slug":"deepseek-v2-chat","modelId":"deepseek-v2-chat","modelSlug":"deepseek-v2-chat","name":"DeepSeek-V2-Chat","organizationSlug":"deepseek","n":33,"rankEligible":true,"score":117.83,"scoreLo":114.07,"scoreHi":121.16,"modelScoreStateId":"deepseek-v2-chat::default","modelScoreConfigSlug":"deepseek-v2-chat","modelScoreStateName":"DeepSeek-V2-Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":529.1456,"rankLo":504.0,"rankHi":553.0},{"slug":"phi-3-5-moe-instruct","modelId":"phi-3-5-moe-instruct","modelSlug":"phi-3-5-moe-instruct","name":"Phi-3.5-MoE-instruct","organizationSlug":"microsoft","n":9,"rankEligible":true,"score":117.83,"scoreLo":113.43,"scoreHi":121.87,"modelScoreStateId":"phi-3-5-moe-instruct::default","modelScoreConfigSlug":"phi-3-5-moe-instruct","modelScoreStateName":"Phi-3.5-MoE-instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":529.1214,"rankLo":486.0,"rankHi":567.0},{"slug":"nova-lite-v1-0","modelId":"nova-lite-v1-0","modelSlug":"nova-lite-v1-0","name":"Nova Lite","organizationSlug":"amazon","n":27,"rankEligible":true,"score":117.78,"scoreLo":113.59,"scoreHi":121.47,"modelScoreStateId":"nova-lite-v1-0::default","modelScoreConfigSlug":"nova-lite-v1-0","modelScoreStateName":"Nova Lite","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":529.7553,"rankLo":495.0,"rankHi":563.0},{"slug":"kimi-linear-48b-a3b-base","modelId":"kimi-linear-48b-a3b-base","modelSlug":"kimi-linear-48b-a3b-base","name":"Kimi Linear 48B A3B Base","organizationSlug":"moonshot-ai","n":12,"rankEligible":true,"score":117.75,"scoreLo":112.87,"scoreHi":122.32,"modelScoreStateId":"kimi-linear-48b-a3b-base::default","modelScoreConfigSlug":"kimi-linear-48b-a3b-base","modelScoreStateName":"Kimi Linear 48B A3B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":529.8856,"rankLo":481.0,"rankHi":575.0},{"slug":"hermes-3-llama-3-1-70b","modelId":"hermes-3-llama-3-1-70b","modelSlug":"hermes-3-llama-3-1-70b","name":"Hermes 3 Llama 3.1 70B","organizationSlug":"nous-research","n":9,"rankEligible":true,"score":117.73,"scoreLo":113.2,"scoreHi":121.85,"modelScoreStateId":"hermes-3-llama-3-1-70b::default","modelScoreConfigSlug":"hermes-3-llama-3-1-70b","modelScoreStateName":"Hermes 3 Llama 3.1 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":530.2006,"rankLo":489.0,"rankHi":570.0},{"slug":"llama-3-70b-instruct","modelId":"llama-3-70b-instruct","modelSlug":"llama-3-70b-instruct","name":"Llama 3 70B Instruct","organizationSlug":"meta","n":61,"rankEligible":true,"score":117.72,"scoreLo":114.32,"scoreHi":120.66,"modelScoreStateId":"llama-3-70b-instruct::default","modelScoreConfigSlug":"llama-3-70b-instruct","modelScoreStateName":"Llama 3 70B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":530.585,"rankLo":513.0,"rankHi":548.0},{"slug":"claude-3-sonnet-20240229","modelId":"claude-3-sonnet-20240229","modelSlug":"claude-3-sonnet-20240229","name":"Claude 3 Sonnet","organizationSlug":"anthropic","n":50,"rankEligible":true,"score":117.5,"scoreLo":113.89,"scoreHi":120.48,"modelScoreStateId":"claude-3-sonnet-20240229::default","modelScoreConfigSlug":"claude-3-sonnet-20240229","modelScoreStateName":"Claude 3 Sonnet","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":533.03,"rankLo":513.0,"rankHi":552.0},{"slug":"k2-horizon-0-9b","modelId":"k2-horizon-0-9b","modelSlug":"k2-horizon-0-9b","name":"K2 Horizon 0.9B","organizationSlug":"llm360","n":9,"rankEligible":true,"score":117.37,"scoreLo":110.95,"scoreHi":122.88,"modelScoreStateId":"k2-horizon-0-9b::enabled","modelScoreConfigSlug":"k2-horizon-0-9b-thinking","modelScoreStateName":"K2 Horizon 0.9B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":534.1292,"rankLo":470.975,"rankHi":601.0},{"slug":"qwen3-8b-base","modelId":"qwen3-8b-base","modelSlug":"qwen3-8b-base","name":"Qwen3 8B Base","organizationSlug":"alibaba","n":9,"rankEligible":true,"score":117.33,"scoreLo":112.95,"scoreHi":121.2,"modelScoreStateId":"qwen3-8b-base::default","modelScoreConfigSlug":"qwen3-8b-base","modelScoreStateName":"Qwen3 8B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":534.9361,"rankLo":500.0,"rankHi":569.0},{"slug":"gemma-2-27b-it","modelId":"gemma-2-27b-it","modelSlug":"gemma-2-27b-it","name":"Gemma 2 27B IT","organizationSlug":"google","n":41,"rankEligible":true,"score":117.31,"scoreLo":113.71,"scoreHi":120.51,"modelScoreStateId":"gemma-2-27b-it::default","modelScoreConfigSlug":"gemma-2-27b-it","modelScoreStateName":"Gemma 2 27B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":535.1264,"rankLo":514.0,"rankHi":555.0},{"slug":"qwen2-5-coder-32b","modelId":"qwen2-5-coder-32b","modelSlug":"qwen2-5-coder-32b","name":"Qwen2.5-Coder-32B","organizationSlug":"alibaba","n":14,"rankEligible":true,"score":117.28,"scoreLo":112.27,"scoreHi":121.81,"modelScoreStateId":"qwen2-5-coder-32b::default","modelScoreConfigSlug":"qwen2-5-coder-32b","modelScoreStateName":"Qwen2.5-Coder-32B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":535.3689,"rankLo":487.0,"rankHi":580.0},{"slug":"llama-3-1-nemotron-51b-instruct","modelId":"llama-3-1-nemotron-51b-instruct","modelSlug":"llama-3-1-nemotron-51b-instruct","name":"Llama 3.1 Nemotron 51B Instruct","organizationSlug":"nvidia","n":6,"rankEligible":true,"score":117.19,"scoreLo":109.3,"scoreHi":124.82,"modelScoreStateId":"llama-3-1-nemotron-51b-instruct::default","modelScoreConfigSlug":"llama-3-1-nemotron-51b-instruct","modelScoreStateName":"Llama 3.1 Nemotron 51B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":535.7703,"rankLo":443.0,"rankHi":621.0},{"slug":"c4ai-command-r-08-2024","modelId":"c4ai-command-r-08-2024","modelSlug":"c4ai-command-r-08-2024","name":"C4AI Command R 08-2024","organizationSlug":"cohere","n":6,"rankEligible":true,"score":117.03,"scoreLo":108.48,"scoreHi":123.91,"modelScoreStateId":"c4ai-command-r-08-2024::default","modelScoreConfigSlug":"c4ai-command-r-08-2024","modelScoreStateName":"C4AI Command R 08-2024","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":537.8975,"rankLo":455.0,"rankHi":629.0},{"slug":"claude-1-3","modelId":"claude-1-3","modelSlug":"claude-1-3","name":"Claude 1.3","organizationSlug":"anthropic","n":8,"rankEligible":true,"score":116.94,"scoreLo":111.27,"scoreHi":122.49,"modelScoreStateId":"claude-1-3::default","modelScoreConfigSlug":"claude-1-3","modelScoreStateName":"Claude 1.3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":539.1006,"rankLo":475.0,"rankHi":593.0},{"slug":"llama-3-1-70b","modelId":"llama-3-1-70b","modelSlug":"llama-3-1-70b","name":"Llama 3.1 70B","organizationSlug":"meta","n":27,"rankEligible":true,"score":116.94,"scoreLo":113.02,"scoreHi":120.39,"modelScoreStateId":"llama-3-1-70b::default","modelScoreConfigSlug":"llama-3-1-70b","modelScoreStateName":"Llama 3.1 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":539.3419,"rankLo":510.0,"rankHi":569.0},{"slug":"mistral-small-24b-base-2501","modelId":"mistral-small-24b-base-2501","modelSlug":"mistral-small-24b-base-2501","name":"Mistral Small 3 24B Base 2501","organizationSlug":"mistral-ai","n":8,"rankEligible":true,"score":116.87,"scoreLo":111.9,"scoreHi":121.33,"modelScoreStateId":"mistral-small-24b-base-2501::default","modelScoreConfigSlug":"mistral-small-24b-base-2501","modelScoreStateName":"Mistral Small 3 24B Base 2501","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":540.13,"rankLo":493.975,"rankHi":586.0},{"slug":"nemotron-nano-9b-v2-base","modelId":"nemotron-nano-9b-v2-base","modelSlug":"nemotron-nano-9b-v2-base","name":"Nemotron Nano 9B V2 Base","organizationSlug":"nvidia","n":12,"rankEligible":true,"score":116.75,"scoreLo":111.95,"scoreHi":121.2,"modelScoreStateId":"nemotron-nano-9b-v2-base::default","modelScoreConfigSlug":"nemotron-nano-9b-v2-base","modelScoreStateName":"Nemotron Nano 9B V2 Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":541.61,"rankLo":499.0,"rankHi":583.025},{"slug":"granite-4-1-3b","modelId":"granite-4-1-3b","modelSlug":"granite-4-1-3b","name":"Granite 4.1 3B","organizationSlug":"ibm","n":8,"rankEligible":true,"score":116.66,"scoreLo":108.49,"scoreHi":123.51,"modelScoreStateId":"granite-4-1-3b::default","modelScoreConfigSlug":"granite-4-1-3b","modelScoreStateName":"Granite 4.1 3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":542.4922,"rankLo":460.0,"rankHi":632.0},{"slug":"ministral-3-14b-base-2512","modelId":"ministral-3-14b-base-2512","modelSlug":"ministral-3-14b-base-2512","name":"Ministral 3 14B Base 2512","organizationSlug":"mistral-ai","n":10,"rankEligible":true,"score":116.63,"scoreLo":111.99,"scoreHi":120.86,"modelScoreStateId":"ministral-3-14b-base-2512::default","modelScoreConfigSlug":"ministral-3-14b-base-2512","modelScoreStateName":"Ministral 3 14B Base 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":542.9614,"rankLo":505.0,"rankHi":584.0},{"slug":"jamba-reasoning-3b","modelId":"jamba-reasoning-3b","modelSlug":"jamba-reasoning-3b","name":"Jamba Reasoning 3B","organizationSlug":"ai21-labs","n":7,"rankEligible":true,"score":116.53,"scoreLo":110.16,"scoreHi":122.03,"modelScoreStateId":"jamba-reasoning-3b::default","modelScoreConfigSlug":"jamba-reasoning-3b","modelScoreStateName":"Jamba Reasoning 3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":544.0911,"rankLo":485.0,"rankHi":608.0},{"slug":"mistral-small-3-1-24b-base-2503","modelId":"mistral-small-3-1-24b-base-2503","modelSlug":"mistral-small-3-1-24b-base-2503","name":"Mistral Small 3.1 24B Base 2503","organizationSlug":"mistral-ai","n":11,"rankEligible":true,"score":116.48,"scoreLo":111.59,"scoreHi":120.92,"modelScoreStateId":"mistral-small-3-1-24b-base-2503::default","modelScoreConfigSlug":"mistral-small-3-1-24b-base-2503","modelScoreStateName":"Mistral Small 3.1 24B Base 2503","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":544.6769,"rankLo":501.975,"rankHi":588.0},{"slug":"wizardlm-2-8x22b","modelId":"wizardlm-2-8x22b","modelSlug":"wizardlm-2-8x22b","name":"WizardLM-2 8x22B","organizationSlug":"microsoft","n":6,"rankEligible":true,"score":116.44,"scoreLo":110.41,"scoreHi":121.94,"modelScoreStateId":"wizardlm-2-8x22b::default","modelScoreConfigSlug":"wizardlm-2-8x22b","modelScoreStateName":"WizardLM-2 8x22B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":545.3264,"rankLo":485.0,"rankHi":606.0},{"slug":"gemini-1-5-flash-001","modelId":"gemini-1-5-flash-001","modelSlug":"gemini-1-5-flash-001","name":"Gemini 1.5 Flash 001","organizationSlug":"google","n":6,"rankEligible":true,"score":116.29,"scoreLo":109.54,"scoreHi":122.16,"modelScoreStateId":"gemini-1-5-flash-001::default","modelScoreConfigSlug":"gemini-1-5-flash-001","modelScoreStateName":"Gemini 1.5 Flash 001","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":547.0975,"rankLo":482.0,"rankHi":616.0},{"slug":"palm-2-l","modelId":"palm-2-l","modelSlug":"palm-2-l","name":"PaLM 2-L","organizationSlug":"google","n":12,"rankEligible":true,"score":116.09,"scoreLo":109.5,"scoreHi":122.1,"modelScoreStateId":"palm-2-l::default","modelScoreConfigSlug":"palm-2-l","modelScoreStateName":"PaLM 2-L","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":549.3047,"rankLo":479.0,"rankHi":616.0},{"slug":"qwen2-5-14b","modelId":"qwen2-5-14b","modelSlug":"qwen2-5-14b","name":"Qwen2.5 14B","organizationSlug":"alibaba","n":16,"rankEligible":true,"score":116.06,"scoreLo":111.25,"scoreHi":120.38,"modelScoreStateId":"qwen2-5-14b::default","modelScoreConfigSlug":"qwen2-5-14b","modelScoreStateName":"Qwen2.5 14B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":549.5039,"rankLo":510.0,"rankHi":591.0},{"slug":"mistral-large-2402","modelId":"mistral-large-2402","modelSlug":"mistral-large-2402","name":"Mistral Large 1.0","organizationSlug":"mistral-ai","n":26,"rankEligible":true,"score":115.84,"scoreLo":111.93,"scoreHi":119.23,"modelScoreStateId":"mistral-large-2402::default","modelScoreConfigSlug":"mistral-large-2402","modelScoreStateName":"Mistral Large 1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":551.9358,"rankLo":526.0,"rankHi":580.0},{"slug":"qwen2-5-7b-instruct-1m","modelId":"qwen2-5-7b-instruct-1m","modelSlug":"qwen2-5-7b-instruct-1m","name":"Qwen2.5 7B Instruct 1M","organizationSlug":"alibaba","n":10,"rankEligible":true,"score":115.74,"scoreLo":110.75,"scoreHi":120.2,"modelScoreStateId":"qwen2-5-7b-instruct-1m::default","modelScoreConfigSlug":"qwen2-5-7b-instruct-1m","modelScoreStateName":"Qwen2.5 7B Instruct 1M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":553.3728,"rankLo":512.0,"rankHi":597.0},{"slug":"qwen1-5-110b","modelId":"qwen1-5-110b","modelSlug":"qwen1-5-110b","name":"Qwen1.5 110B","organizationSlug":"alibaba","n":16,"rankEligible":true,"score":115.69,"scoreLo":110.92,"scoreHi":120.08,"modelScoreStateId":"qwen1-5-110b::default","modelScoreConfigSlug":"qwen1-5-110b","modelScoreStateName":"Qwen1.5 110B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":553.8453,"rankLo":516.0,"rankHi":594.025},{"slug":"grok-1-5","modelId":"grok-1-5","modelSlug":"grok-1-5","name":"Grok 1.5","organizationSlug":"xai","n":6,"rankEligible":true,"score":115.66,"scoreLo":109.75,"scoreHi":120.93,"modelScoreStateId":"grok-1-5::default","modelScoreConfigSlug":"grok-1-5","modelScoreStateName":"Grok 1.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":554.3642,"rankLo":501.0,"rankHi":610.0},{"slug":"apertus-70b-instruct-2509","modelId":"apertus-70b-instruct-2509","modelSlug":"apertus-70b-instruct-2509","name":"Apertus 70B Instruct","organizationSlug":"swiss-ai","n":8,"rankEligible":true,"score":115.64,"scoreLo":103.51,"scoreHi":125.61,"modelScoreStateId":"apertus-70b-instruct-2509::default","modelScoreConfigSlug":"apertus-70b-instruct-2509","modelScoreStateName":"Apertus 70B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":553.5856,"rankLo":432.0,"rankHi":682.0},{"slug":"minicpm-v-4-6","modelId":"minicpm-v-4-6","modelSlug":"minicpm-v-4-6","name":"MiniCPM-V 4.6","organizationSlug":"openbmb","n":9,"rankEligible":true,"score":115.33,"scoreLo":107.13,"scoreHi":122.22,"modelScoreStateId":"minicpm-v-4-6::default","modelScoreConfigSlug":"minicpm-v-4-6","modelScoreStateName":"MiniCPM-V 4.6","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":558.435,"rankLo":478.0,"rankHi":647.0},{"slug":"minicpm-o-2-6","modelId":"minicpm-o-2-6","modelSlug":"minicpm-o-2-6","name":"MiniCPM-o 2.6","organizationSlug":"openbmb","n":11,"rankEligible":true,"score":115.32,"scoreLo":109.82,"scoreHi":120.28,"modelScoreStateId":"minicpm-o-2-6::default","modelScoreConfigSlug":"minicpm-o-2-6","modelScoreStateName":"MiniCPM-o 2.6","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":558.3722,"rankLo":511.0,"rankHi":610.025},{"slug":"qwen1-5-110b-chat","modelId":"qwen1-5-110b-chat","modelSlug":"qwen1-5-110b-chat","name":"Qwen1.5 110B Chat","organizationSlug":"alibaba","n":31,"rankEligible":true,"score":115.26,"scoreLo":111.04,"scoreHi":118.86,"modelScoreStateId":"qwen1-5-110b-chat::default","modelScoreConfigSlug":"qwen1-5-110b-chat","modelScoreStateName":"Qwen1.5 110B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":558.5564,"rankLo":533.0,"rankHi":586.0},{"slug":"mixtral-8x22b-instruct-v0-1","modelId":"mixtral-8x22b-instruct-v0-1","modelSlug":"mixtral-8x22b-instruct-v0-1","name":"Mixtral 8x22B Instruct","organizationSlug":"mistral-ai","n":60,"rankEligible":true,"score":115.19,"scoreLo":111.51,"scoreHi":118.44,"modelScoreStateId":"mixtral-8x22b-instruct-v0-1::default","modelScoreConfigSlug":"mixtral-8x22b-instruct-v0-1","modelScoreStateName":"Mixtral 8x22B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":559.3053,"rankLo":540.0,"rankHi":580.0},{"slug":"deepseek-v2","modelId":"deepseek-v2","modelSlug":"deepseek-v2","name":"DeepSeek-V2","organizationSlug":"deepseek","n":29,"rankEligible":true,"score":115.08,"scoreLo":110.72,"scoreHi":118.94,"modelScoreStateId":"deepseek-v2::default","modelScoreConfigSlug":"deepseek-v2","modelScoreStateName":"DeepSeek-V2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":560.9303,"rankLo":529.0,"rankHi":593.0},{"slug":"smaug-72b-v0-1","modelId":"smaug-72b-v0-1","modelSlug":"smaug-72b-v0-1","name":"Smaug-72B-v0.1","organizationSlug":"abacus-ai","n":13,"rankEligible":true,"score":114.98,"scoreLo":110.26,"scoreHi":119.14,"modelScoreStateId":"smaug-72b-v0-1::default","modelScoreConfigSlug":"smaug-72b-v0-1","modelScoreStateName":"Smaug-72B-v0.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":562.3056,"rankLo":528.0,"rankHi":601.0},{"slug":"gemma-3-27b-pt","modelId":"gemma-3-27b-pt","modelSlug":"gemma-3-27b-pt","name":"Gemma 3 27B PT","organizationSlug":"google","n":22,"rankEligible":true,"score":114.83,"scoreLo":110.3,"scoreHi":118.68,"modelScoreStateId":"gemma-3-27b-pt::default","modelScoreConfigSlug":"gemma-3-27b-pt","modelScoreStateName":"Gemma 3 27B PT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":563.9228,"rankLo":534.0,"rankHi":596.0},{"slug":"gpt-4-vision-preview","modelId":"gpt-4-vision-preview","modelSlug":"gpt-4-vision-preview","name":"GPT-4 Turbo with Vision","organizationSlug":"openai","n":4,"rankEligible":true,"score":114.82,"scoreLo":107.96,"scoreHi":120.64,"modelScoreStateId":"gpt-4-vision-preview::default","modelScoreConfigSlug":"gpt-4-vision-preview","modelScoreStateName":"GPT-4 Turbo with Vision","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":564.6492,"rankLo":502.0,"rankHi":637.0},{"slug":"c4ai-command-r-plus-08-2024","modelId":"c4ai-command-r-plus-08-2024","modelSlug":"c4ai-command-r-plus-08-2024","name":"C4AI Command R+ 08-2024","organizationSlug":"cohere","n":13,"rankEligible":true,"score":114.76,"scoreLo":109.44,"scoreHi":119.32,"modelScoreStateId":"c4ai-command-r-plus-08-2024::default","modelScoreConfigSlug":"c4ai-command-r-plus-08-2024","modelScoreStateName":"C4AI Command R+ 08-2024","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":565.0544,"rankLo":522.0,"rankHi":614.0},{"slug":"llama-4-scout-17b-16e","modelId":"llama-4-scout-17b-16e","modelSlug":"llama-4-scout-17b-16e","name":"Llama 4 Scout","organizationSlug":"meta","n":8,"rankEligible":true,"score":114.68,"scoreLo":109.69,"scoreHi":119.1,"modelScoreStateId":"llama-4-scout-17b-16e::default","modelScoreConfigSlug":"llama-4-scout-17b-16e","modelScoreStateName":"Llama 4 Scout","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":566.0522,"rankLo":526.0,"rankHi":609.0},{"slug":"phi-3-medium-4k-instruct","modelId":"phi-3-medium-4k-instruct","modelSlug":"phi-3-medium-4k-instruct","name":"Phi-3 Medium 4K Instruct","organizationSlug":"microsoft","n":18,"rankEligible":true,"score":114.66,"scoreLo":110.27,"scoreHi":118.52,"modelScoreStateId":"phi-3-medium-4k-instruct::default","modelScoreConfigSlug":"phi-3-medium-4k-instruct","modelScoreStateName":"Phi-3 Medium 4K Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":566.0572,"rankLo":535.0,"rankHi":600.0},{"slug":"aya-expanse-32b","modelId":"aya-expanse-32b","modelSlug":"aya-expanse-32b","name":"Aya Expanse 32B","organizationSlug":"cohere","n":16,"rankEligible":true,"score":114.56,"scoreLo":109.22,"scoreHi":119.22,"modelScoreStateId":"aya-expanse-32b::default","modelScoreConfigSlug":"aya-expanse-32b","modelScoreStateName":"Aya Expanse 32B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":567.4819,"rankLo":523.975,"rankHi":617.0},{"slug":"aria","modelId":"aria","modelSlug":"aria","name":"Aria","organizationSlug":"rhymes-ai","n":10,"rankEligible":true,"score":114.5,"scoreLo":109.32,"scoreHi":119.13,"modelScoreStateId":"aria::default","modelScoreConfigSlug":"aria","modelScoreStateName":"Aria","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":568.1206,"rankLo":524.0,"rankHi":618.0},{"slug":"llama-3-70b","modelId":"llama-3-70b","modelSlug":"llama-3-70b","name":"Llama 3 70B","organizationSlug":"meta","n":39,"rankEligible":true,"score":114.42,"scoreLo":110.21,"scoreHi":117.94,"modelScoreStateId":"llama-3-70b::default","modelScoreConfigSlug":"llama-3-70b","modelScoreStateName":"Llama 3 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":568.8808,"rankLo":542.0,"rankHi":597.0},{"slug":"qwen-72b-chat","modelId":"qwen-72b-chat","modelSlug":"qwen-72b-chat","name":"Qwen 72B Chat","organizationSlug":"alibaba","n":6,"rankEligible":true,"score":114.33,"scoreLo":108.13,"scoreHi":119.8,"modelScoreStateId":"qwen-72b-chat::default","modelScoreConfigSlug":"qwen-72b-chat","modelScoreStateName":"Qwen 72B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":570.3572,"rankLo":514.0,"rankHi":628.0},{"slug":"gemini-1-5-flash-8b","modelId":"gemini-1-5-flash-8b","modelSlug":"gemini-1-5-flash-8b","name":"Gemini 1.5 Flash-8B","organizationSlug":"google","n":8,"rankEligible":true,"score":114.27,"scoreLo":108.96,"scoreHi":118.97,"modelScoreStateId":"gemini-1-5-flash-8b::default","modelScoreConfigSlug":"gemini-1-5-flash-8b","modelScoreStateName":"Gemini 1.5 Flash-8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":571.0269,"rankLo":527.0,"rankHi":619.025},{"slug":"lfm2-5-vl-3b","modelId":"lfm2-5-vl-3b","modelSlug":"lfm2-5-vl-3b","name":"LFM2.5-VL-3B","organizationSlug":"liquid-ai","n":48,"rankEligible":true,"score":114.12,"scoreLo":109.51,"scoreHi":118.04,"modelScoreStateId":"lfm2-5-vl-3b::default","modelScoreConfigSlug":"lfm2-5-vl-3b","modelScoreStateName":"LFM2.5-VL-3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":572.5953,"rankLo":542.0,"rankHi":606.0},{"slug":"claude-2-1","modelId":"claude-2-1","modelSlug":"claude-2-1","name":"Claude 2.1","organizationSlug":"anthropic","n":11,"rankEligible":true,"score":113.97,"scoreLo":109.07,"scoreHi":118.3,"modelScoreStateId":"claude-2-1::default","modelScoreConfigSlug":"claude-2-1","modelScoreStateName":"Claude 2.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":574.5003,"rankLo":536.0,"rankHi":617.0},{"slug":"lfm2-24b-a2b","modelId":"lfm2-24b-a2b","modelSlug":"lfm2-24b-a2b","name":"LFM2-24B-A2B","organizationSlug":"liquid-ai","n":5,"rankEligible":true,"score":113.94,"scoreLo":97.03,"scoreHi":128.04,"modelScoreStateId":"lfm2-24b-a2b::default","modelScoreConfigSlug":"lfm2-24b-a2b","modelScoreStateName":"LFM2-24B-A2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":571.5697,"rankLo":399.0,"rankHi":736.025},{"slug":"qwen2-5-7b-instruct","modelId":"qwen2-5-7b-instruct","modelSlug":"qwen2-5-7b-instruct","name":"Qwen2.5 7B Instruct","organizationSlug":"alibaba","n":52,"rankEligible":true,"score":113.82,"scoreLo":109.79,"scoreHi":117.29,"modelScoreStateId":"qwen2-5-7b-instruct::default","modelScoreConfigSlug":"qwen2-5-7b-instruct","modelScoreStateName":"Qwen2.5 7B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":576.1453,"rankLo":553.0,"rankHi":600.0},{"slug":"jamba-large-1-6","modelId":"jamba-large-1-6","modelSlug":"jamba-large-1-6","name":"Jamba Large 1.6","organizationSlug":"ai21-labs","n":9,"rankEligible":true,"score":113.81,"scoreLo":108.45,"scoreHi":118.54,"modelScoreStateId":"jamba-large-1-6::default","modelScoreConfigSlug":"jamba-large-1-6","modelScoreStateName":"Jamba Large 1.6","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":576.7722,"rankLo":533.0,"rankHi":625.0},{"slug":"nova-micro-v1-0","modelId":"nova-micro-v1-0","modelSlug":"nova-micro-v1-0","name":"Nova Micro","organizationSlug":"amazon","n":24,"rankEligible":true,"score":113.77,"scoreLo":108.56,"scoreHi":118.22,"modelScoreStateId":"nova-micro-v1-0::default","modelScoreConfigSlug":"nova-micro-v1-0","modelScoreStateName":"Nova Micro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":577.1419,"rankLo":537.0,"rankHi":621.0},{"slug":"phi-3-small-8k-instruct","modelId":"phi-3-small-8k-instruct","modelSlug":"phi-3-small-8k-instruct","name":"Phi-3-Small-8K-Instruct","organizationSlug":"microsoft","n":10,"rankEligible":true,"score":113.74,"scoreLo":108.87,"scoreHi":117.98,"modelScoreStateId":"phi-3-small-8k-instruct::default","modelScoreConfigSlug":"phi-3-small-8k-instruct","modelScoreStateName":"Phi-3-Small-8K-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":577.4756,"rankLo":539.975,"rankHi":620.0},{"slug":"mixtral-8x22b-v0-1","modelId":"mixtral-8x22b-v0-1","modelSlug":"mixtral-8x22b-v0-1","name":"Mixtral 8x22B","organizationSlug":"mistral-ai","n":47,"rankEligible":true,"score":113.58,"scoreLo":109.48,"scoreHi":117.11,"modelScoreStateId":"mixtral-8x22b-v0-1::default","modelScoreConfigSlug":"mixtral-8x22b-v0-1","modelScoreStateName":"Mixtral 8x22B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":579.3144,"rankLo":555.0,"rankHi":603.0},{"slug":"grok-1-5v","modelId":"grok-1-5v","modelSlug":"grok-1-5v","name":"Grok 1.5V","organizationSlug":"xai","n":5,"rankEligible":true,"score":113.57,"scoreLo":105.99,"scoreHi":120.66,"modelScoreStateId":"grok-1-5v::default","modelScoreConfigSlug":"grok-1-5v","modelScoreStateName":"Grok 1.5V","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":579.4831,"rankLo":505.0,"rankHi":656.025},{"slug":"phi-3-medium-128k-instruct","modelId":"phi-3-medium-128k-instruct","modelSlug":"phi-3-medium-128k-instruct","name":"Phi-3 Medium 128K Instruct","organizationSlug":"microsoft","n":14,"rankEligible":true,"score":113.54,"scoreLo":108.78,"scoreHi":117.83,"modelScoreStateId":"phi-3-medium-128k-instruct::default","modelScoreConfigSlug":"phi-3-medium-128k-instruct","modelScoreStateName":"Phi-3 Medium 128K Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":579.7931,"rankLo":543.975,"rankHi":618.0},{"slug":"deepseek-vl2","modelId":"deepseek-vl2","modelSlug":"deepseek-vl2","name":"DeepSeek-VL2","organizationSlug":"deepseek","n":8,"rankEligible":true,"score":113.5,"scoreLo":107.09,"scoreHi":118.87,"modelScoreStateId":"deepseek-vl2::default","modelScoreConfigSlug":"deepseek-vl2","modelScoreStateName":"DeepSeek-VL2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":580.5475,"rankLo":524.0,"rankHi":642.0},{"slug":"qwen3-1-7b","modelId":"qwen3-1-7b","modelSlug":"qwen3-1-7b","name":"Qwen3 1.7B","organizationSlug":"alibaba","n":31,"rankEligible":true,"score":113.45,"scoreLo":109.33,"scoreHi":117.1,"modelScoreStateId":"qwen3-1-7b::enabled","modelScoreConfigSlug":"qwen3-1-7b-thinking","modelScoreStateName":"Qwen3 1.7B (thinking)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":580.8914,"rankLo":555.0,"rankHi":607.0},{"slug":"qwen2-5-vl-3b-instruct","modelId":"qwen2-5-vl-3b-instruct","modelSlug":"qwen2-5-vl-3b-instruct","name":"Qwen2.5-VL-3B-Instruct","organizationSlug":"alibaba","n":29,"rankEligible":true,"score":113.41,"scoreLo":108.5,"scoreHi":117.65,"modelScoreStateId":"qwen2-5-vl-3b-instruct::default","modelScoreConfigSlug":"qwen2-5-vl-3b-instruct","modelScoreStateName":"Qwen2.5-VL-3B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":581.5494,"rankLo":545.0,"rankHi":620.0},{"slug":"lfm2-5-1-2b-thinking","modelId":"lfm2-5-1-2b-thinking","modelSlug":"lfm2-5-1-2b-thinking","name":"LFM2.5-1.2B-Thinking","organizationSlug":"liquid-ai","n":5,"rankEligible":true,"score":113.36,"scoreLo":96.26,"scoreHi":128.5,"modelScoreStateId":"lfm2-5-1-2b-thinking::default","modelScoreConfigSlug":"lfm2-5-1-2b-thinking","modelScoreStateName":"LFM2.5-1.2B-Thinking","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":577.9411,"rankLo":395.0,"rankHi":744.025},{"slug":"qwen3-vl-2b-instruct","modelId":"qwen3-vl-2b-instruct","modelSlug":"qwen3-vl-2b-instruct","name":"Qwen3 VL 2B Instruct","organizationSlug":"alibaba","n":43,"rankEligible":true,"score":113.16,"scoreLo":108.99,"scoreHi":116.75,"modelScoreStateId":"qwen3-vl-2b-instruct::default","modelScoreConfigSlug":"qwen3-vl-2b-instruct","modelScoreStateName":"Qwen3 VL 2B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":584.6678,"rankLo":558.0,"rankHi":611.0},{"slug":"qwen3-4b-base","modelId":"qwen3-4b-base","modelSlug":"qwen3-4b-base","name":"Qwen3 4B Base","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":113.14,"scoreLo":108.15,"scoreHi":117.41,"modelScoreStateId":"qwen3-4b-base::default","modelScoreConfigSlug":"qwen3-4b-base","modelScoreStateName":"Qwen3 4B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":584.9278,"rankLo":546.0,"rankHi":625.0},{"slug":"yi-1-5-34b-chat","modelId":"yi-1-5-34b-chat","modelSlug":"yi-1-5-34b-chat","name":"Yi 1.5 34B Chat","organizationSlug":"01-ai","n":27,"rankEligible":true,"score":113.04,"scoreLo":108.58,"scoreHi":116.73,"modelScoreStateId":"yi-1-5-34b-chat::default","modelScoreConfigSlug":"yi-1-5-34b-chat","modelScoreStateName":"Yi 1.5 34B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":586.09,"rankLo":558.0,"rankHi":614.0},{"slug":"qwen2-5-coder-14b","modelId":"qwen2-5-coder-14b","modelSlug":"qwen2-5-coder-14b","name":"Qwen2.5-Coder-14B","organizationSlug":"alibaba","n":20,"rankEligible":true,"score":113.02,"scoreLo":108.33,"scoreHi":117.35,"modelScoreStateId":"qwen2-5-coder-14b::default","modelScoreConfigSlug":"qwen2-5-coder-14b","modelScoreStateName":"Qwen2.5-Coder-14B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":586.3144,"rankLo":549.0,"rankHi":622.0},{"slug":"claude-2-0","modelId":"claude-2-0","modelSlug":"claude-2-0","name":"Claude 2","organizationSlug":"anthropic","n":12,"rankEligible":true,"score":112.95,"scoreLo":107.68,"scoreHi":117.47,"modelScoreStateId":"claude-2-0::default","modelScoreConfigSlug":"claude-2-0","modelScoreStateName":"Claude 2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":587.1733,"rankLo":545.0,"rankHi":629.0},{"slug":"pixtral-12b-2409","modelId":"pixtral-12b-2409","modelSlug":"pixtral-12b-2409","name":"Pixtral 12B","organizationSlug":"mistral-ai","n":20,"rankEligible":true,"score":112.8,"scoreLo":107.69,"scoreHi":117.16,"modelScoreStateId":"pixtral-12b-2409::default","modelScoreConfigSlug":"pixtral-12b-2409","modelScoreStateName":"Pixtral 12B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":588.9794,"rankLo":549.0,"rankHi":629.0},{"slug":"nemotron-h-4b-instruct-128k","modelId":"nemotron-h-4b-instruct-128k","modelSlug":"nemotron-h-4b-instruct-128k","name":"Nemotron-H 4B Instruct 128K","organizationSlug":"nvidia","n":7,"rankEligible":true,"score":112.74,"scoreLo":105.88,"scoreHi":119.59,"modelScoreStateId":"nemotron-h-4b-instruct-128k::default","modelScoreConfigSlug":"nemotron-h-4b-instruct-128k","modelScoreStateName":"Nemotron-H 4B Instruct 128K","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":589.8261,"rankLo":516.0,"rankHi":657.0},{"slug":"qwen2-vl-7b-instruct","modelId":"qwen2-vl-7b-instruct","modelSlug":"qwen2-vl-7b-instruct","name":"Qwen2 VL 7B Instruct","organizationSlug":"alibaba","n":33,"rankEligible":true,"score":112.52,"scoreLo":107.82,"scoreHi":116.43,"modelScoreStateId":"qwen2-vl-7b-instruct::default","modelScoreConfigSlug":"qwen2-vl-7b-instruct","modelScoreStateName":"Qwen2 VL 7B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":592.7378,"rankLo":562.0,"rankHi":623.0},{"slug":"qwen1-5-72b-chat","modelId":"qwen1-5-72b-chat","modelSlug":"qwen1-5-72b-chat","name":"Qwen1.5 72B Chat","organizationSlug":"alibaba","n":32,"rankEligible":true,"score":112.38,"scoreLo":107.94,"scoreHi":116.04,"modelScoreStateId":"qwen1-5-72b-chat::default","modelScoreConfigSlug":"qwen1-5-72b-chat","modelScoreStateName":"Qwen1.5 72B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":594.5047,"rankLo":567.0,"rankHi":621.0},{"slug":"qwen2-57b-a14b-instruct","modelId":"qwen2-57b-a14b-instruct","modelSlug":"qwen2-57b-a14b-instruct","name":"Qwen2-57B-A14B Instruct","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":112.06,"scoreLo":106.93,"scoreHi":116.46,"modelScoreStateId":"qwen2-57b-a14b-instruct::default","modelScoreConfigSlug":"qwen2-57b-a14b-instruct","modelScoreStateName":"Qwen2-57B-A14B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":598.3247,"rankLo":561.0,"rankHi":634.0},{"slug":"ministral-3-8b-base-2512","modelId":"ministral-3-8b-base-2512","modelSlug":"ministral-3-8b-base-2512","name":"Ministral 3 8B Base 2512","organizationSlug":"mistral-ai","n":10,"rankEligible":true,"score":112.01,"scoreLo":106.62,"scoreHi":116.57,"modelScoreStateId":"ministral-3-8b-base-2512::default","modelScoreConfigSlug":"ministral-3-8b-base-2512","modelScoreStateName":"Ministral 3 8B Base 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":598.8994,"rankLo":556.0,"rankHi":643.0},{"slug":"claude-3-haiku-20240307","modelId":"claude-3-haiku-20240307","modelSlug":"claude-3-haiku-20240307","name":"Claude 3 Haiku","organizationSlug":"anthropic","n":61,"rankEligible":true,"score":111.98,"scoreLo":107.81,"scoreHi":115.52,"modelScoreStateId":"claude-3-haiku-20240307::default","modelScoreConfigSlug":"claude-3-haiku-20240307","modelScoreStateName":"Claude 3 Haiku","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":599.4311,"rankLo":578.0,"rankHi":621.0},{"slug":"claude-instant-1-1","modelId":"claude-instant-1-1","modelSlug":"claude-instant-1-1","name":"Claude Instant 1.1","organizationSlug":"anthropic","n":7,"rankEligible":true,"score":111.89,"scoreLo":105.46,"scoreHi":117.61,"modelScoreStateId":"claude-instant-1-1::default","modelScoreConfigSlug":"claude-instant-1-1","modelScoreStateName":"Claude Instant 1.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":600.1764,"rankLo":540.975,"rankHi":656.0},{"slug":"deepseek-coder-33b-instruct","modelId":"deepseek-coder-33b-instruct","modelSlug":"deepseek-coder-33b-instruct","name":"DeepSeek-Coder-Instruct 33B","organizationSlug":"deepseek","n":9,"rankEligible":true,"score":111.89,"scoreLo":105.86,"scoreHi":117.86,"modelScoreStateId":"deepseek-coder-33b-instruct::default","modelScoreConfigSlug":"deepseek-coder-33b-instruct","modelScoreStateName":"DeepSeek-Coder-Instruct 33B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":600.1989,"rankLo":537.0,"rankHi":656.0},{"slug":"mistral-nemo-minitron-8b-instruct","modelId":"mistral-nemo-minitron-8b-instruct","modelSlug":"mistral-nemo-minitron-8b-instruct","name":"Mistral-NeMo-Minitron 8B Instruct","organizationSlug":"nvidia","n":5,"rankEligible":true,"score":111.89,"scoreLo":104.67,"scoreHi":118.44,"modelScoreStateId":"mistral-nemo-minitron-8b-instruct::default","modelScoreConfigSlug":"mistral-nemo-minitron-8b-instruct","modelScoreStateName":"Mistral-NeMo-Minitron 8B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":599.9853,"rankLo":528.975,"rankHi":665.0},{"slug":"nous-hermes-2-yi-34b","modelId":"nous-hermes-2-yi-34b","modelSlug":"nous-hermes-2-yi-34b","name":"Nous Hermes 2 Yi 34B","organizationSlug":"nous-research","n":11,"rankEligible":true,"score":111.73,"scoreLo":106.12,"scoreHi":116.39,"modelScoreStateId":"nous-hermes-2-yi-34b::default","modelScoreConfigSlug":"nous-hermes-2-yi-34b","modelScoreStateName":"Nous Hermes 2 Yi 34B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":602.1919,"rankLo":557.0,"rankHi":646.0},{"slug":"minicpm-v-2-6","modelId":"minicpm-v-2-6","modelSlug":"minicpm-v-2-6","name":"MiniCPM-V 2.6","organizationSlug":"openbmb","n":14,"rankEligible":true,"score":111.72,"scoreLo":106.03,"scoreHi":116.73,"modelScoreStateId":"minicpm-v-2-6::default","modelScoreConfigSlug":"minicpm-v-2-6","modelScoreStateName":"MiniCPM-V 2.6","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":602.1717,"rankLo":554.0,"rankHi":649.0},{"slug":"deepseek-r1-distill-qwen-7b","modelId":"deepseek-r1-distill-qwen-7b","modelSlug":"deepseek-r1-distill-qwen-7b","name":"DeepSeek-R1-Distill-Qwen-7B","organizationSlug":"deepseek","n":8,"rankEligible":true,"score":111.71,"scoreLo":101.58,"scoreHi":119.61,"modelScoreStateId":"deepseek-r1-distill-qwen-7b::default","modelScoreConfigSlug":"deepseek-r1-distill-qwen-7b","modelScoreStateName":"DeepSeek-R1-Distill-Qwen-7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":601.0331,"rankLo":514.0,"rankHi":701.0},{"slug":"nanbeige4-1-3b","modelId":"nanbeige4-1-3b","modelSlug":"nanbeige4-1-3b","name":"Nanbeige4.1-3B","organizationSlug":"nanbeige","n":8,"rankEligible":true,"score":111.58,"scoreLo":97.64,"scoreHi":122.96,"modelScoreStateId":"nanbeige4-1-3b::default","modelScoreConfigSlug":"nanbeige4-1-3b","modelScoreStateName":"Nanbeige4.1-3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":600.545,"rankLo":468.0,"rankHi":730.0},{"slug":"lfm2-8b-a1b","modelId":"lfm2-8b-a1b","modelSlug":"lfm2-8b-a1b","name":"LFM2-8B-A1B","organizationSlug":"liquid-ai","n":7,"rankEligible":true,"score":111.56,"scoreLo":103.8,"scoreHi":118.38,"modelScoreStateId":"lfm2-8b-a1b::default","modelScoreConfigSlug":"lfm2-8b-a1b","modelScoreStateName":"LFM2-8B-A1B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":603.6672,"rankLo":531.0,"rankHi":678.0},{"slug":"yi-34b","modelId":"yi-34b","modelSlug":"yi-34b","name":"Yi-34B","organizationSlug":"01-ai","n":15,"rankEligible":true,"score":111.38,"scoreLo":106.36,"scoreHi":115.88,"modelScoreStateId":"yi-34b::default","modelScoreConfigSlug":"yi-34b","modelScoreStateName":"Yi-34B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":606.5906,"rankLo":564.0,"rankHi":645.0},{"slug":"molmo2-8b","modelId":"molmo2-8b","modelSlug":"molmo2-8b","name":"Molmo 2 8B","organizationSlug":"allen-ai","n":6,"rankEligible":true,"score":111.32,"scoreLo":100.99,"scoreHi":120.34,"modelScoreStateId":"molmo2-8b::default","modelScoreConfigSlug":"molmo2-8b","modelScoreStateName":"Molmo 2 8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":605.3792,"rankLo":506.0,"rankHi":705.025},{"slug":"gemma-2-27b","modelId":"gemma-2-27b","modelSlug":"gemma-2-27b","name":"Gemma 2 27B","organizationSlug":"google","n":28,"rankEligible":true,"score":111.28,"scoreLo":106.58,"scoreHi":115.34,"modelScoreStateId":"gemma-2-27b::default","modelScoreConfigSlug":"gemma-2-27b","modelScoreStateName":"Gemma 2 27B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":608.0958,"rankLo":578.0,"rankHi":637.0},{"slug":"olmo-2-0325-32b-instruct","modelId":"olmo-2-0325-32b-instruct","modelSlug":"olmo-2-0325-32b-instruct","name":"OLMo 2 32B Instruct","organizationSlug":"allen-ai","n":12,"rankEligible":true,"score":111.27,"scoreLo":105.26,"scoreHi":116.49,"modelScoreStateId":"olmo-2-0325-32b-instruct::default","modelScoreConfigSlug":"olmo-2-0325-32b-instruct","modelScoreStateName":"OLMo 2 32B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":607.7894,"rankLo":555.0,"rankHi":659.025},{"slug":"deepseek-vl2-small","modelId":"deepseek-vl2-small","modelSlug":"deepseek-vl2-small","name":"DeepSeek-VL2-Small","organizationSlug":"deepseek","n":7,"rankEligible":true,"score":111.25,"scoreLo":104.57,"scoreHi":117.06,"modelScoreStateId":"deepseek-vl2-small::default","modelScoreConfigSlug":"deepseek-vl2-small","modelScoreStateName":"DeepSeek-VL2-Small","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":607.6644,"rankLo":549.0,"rankHi":666.0},{"slug":"gemma-2-9b-it","modelId":"gemma-2-9b-it","modelSlug":"gemma-2-9b-it","name":"Gemma 2 9B IT","organizationSlug":"google","n":39,"rankEligible":true,"score":111.21,"scoreLo":106.93,"scoreHi":114.89,"modelScoreStateId":"gemma-2-9b-it::default","modelScoreConfigSlug":"gemma-2-9b-it","modelScoreStateName":"Gemma 2 9B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":609.0633,"rankLo":585.0,"rankHi":632.0},{"slug":"mistral-medium-2312","modelId":"mistral-medium-2312","modelSlug":"mistral-medium-2312","name":"Mistral Medium 1.0","organizationSlug":"mistral-ai","n":7,"rankEligible":true,"score":111.14,"scoreLo":105.42,"scoreHi":116.36,"modelScoreStateId":"mistral-medium-2312::default","modelScoreConfigSlug":"mistral-medium","modelScoreStateName":"Mistral Medium 1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":609.3831,"rankLo":559.0,"rankHi":657.0},{"slug":"qwen2-57b-a14b","modelId":"qwen2-57b-a14b","modelSlug":"qwen2-57b-a14b","name":"Qwen2-57B-A14B","organizationSlug":"alibaba","n":17,"rankEligible":true,"score":111.09,"scoreLo":105.79,"scoreHi":115.88,"modelScoreStateId":"qwen2-57b-a14b::default","modelScoreConfigSlug":"qwen2-57b-a14b","modelScoreStateName":"Qwen2-57B-A14B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":610.1242,"rankLo":567.0,"rankHi":650.0},{"slug":"lfm2-vl-3b","modelId":"lfm2-vl-3b","modelSlug":"lfm2-vl-3b","name":"LFM2-VL-3B","organizationSlug":"liquid-ai","n":11,"rankEligible":true,"score":110.91,"scoreLo":104.95,"scoreHi":116.32,"modelScoreStateId":"lfm2-vl-3b::default","modelScoreConfigSlug":"lfm2-vl-3b","modelScoreStateName":"LFM2-VL-3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":612.0931,"rankLo":553.0,"rankHi":665.0},{"slug":"qwen-72b","modelId":"qwen-72b","modelSlug":"qwen-72b","name":"Qwen 72B","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":110.79,"scoreLo":105.22,"scoreHi":115.44,"modelScoreStateId":"qwen-72b::default","modelScoreConfigSlug":"qwen-72b","modelScoreStateName":"Qwen 72B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":613.6486,"rankLo":568.0,"rankHi":655.0},{"slug":"qwen1-5-32b","modelId":"qwen1-5-32b","modelSlug":"qwen1-5-32b","name":"Qwen1.5 32B","organizationSlug":"alibaba","n":26,"rankEligible":true,"score":110.78,"scoreLo":105.92,"scoreHi":114.79,"modelScoreStateId":"qwen1-5-32b::default","modelScoreConfigSlug":"qwen1-5-32b","modelScoreStateName":"Qwen1.5 32B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":614.1111,"rankLo":583.0,"rankHi":644.0},{"slug":"mistral-small-instruct-2409","modelId":"mistral-small-instruct-2409","modelSlug":"mistral-small-instruct-2409","name":"Mistral Small Instruct 2409","organizationSlug":"mistral-ai","n":16,"rankEligible":true,"score":110.73,"scoreLo":105.54,"scoreHi":115.16,"modelScoreStateId":"mistral-small-instruct-2409::default","modelScoreConfigSlug":"mistral-small-instruct-2409","modelScoreStateName":"Mistral Small Instruct 2409","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":614.5856,"rankLo":576.975,"rankHi":650.0},{"slug":"olmo-3-7b-instruct","modelId":"olmo-3-7b-instruct","modelSlug":"olmo-3-7b-instruct","name":"Olmo 3 7B Instruct","organizationSlug":"allen-ai","n":10,"rankEligible":true,"score":110.73,"scoreLo":103.1,"scoreHi":117.36,"modelScoreStateId":"olmo-3-7b-instruct::default","modelScoreConfigSlug":"olmo-3-7b-instruct","modelScoreStateName":"Olmo 3 7B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":613.5078,"rankLo":543.0,"rankHi":680.025},{"slug":"qwen2-5-7b","modelId":"qwen2-5-7b","modelSlug":"qwen2-5-7b","name":"Qwen2.5 7B","organizationSlug":"alibaba","n":19,"rankEligible":true,"score":110.67,"scoreLo":105.37,"scoreHi":115.18,"modelScoreStateId":"qwen2-5-7b::default","modelScoreConfigSlug":"qwen2-5-7b","modelScoreStateName":"Qwen2.5 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":615.2436,"rankLo":577.0,"rankHi":652.0},{"slug":"qwen1-5-32b-chat","modelId":"qwen1-5-32b-chat","modelSlug":"qwen1-5-32b-chat","name":"Qwen1.5 32B Chat","organizationSlug":"alibaba","n":28,"rankEligible":true,"score":110.63,"scoreLo":105.99,"scoreHi":114.7,"modelScoreStateId":"qwen1-5-32b-chat::default","modelScoreConfigSlug":"qwen1-5-32b-chat","modelScoreStateName":"Qwen1.5 32B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":615.9058,"rankLo":588.0,"rankHi":643.0},{"slug":"mistral-small-2402","modelId":"mistral-small-2402","modelSlug":"mistral-small-2402","name":"Mistral Small 1.0","organizationSlug":"mistral-ai","n":11,"rankEligible":true,"score":110.48,"scoreLo":105.22,"scoreHi":115.04,"modelScoreStateId":"mistral-small-2402::default","modelScoreConfigSlug":"mistral-small-2402","modelScoreStateName":"Mistral Small 1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":617.4422,"rankLo":577.0,"rankHi":655.0},{"slug":"gemma-3-4b-it","modelId":"gemma-3-4b-it","modelSlug":"gemma-3-4b-it","name":"Gemma 3 4B IT","organizationSlug":"google","n":55,"rankEligible":true,"score":110.47,"scoreLo":105.8,"scoreHi":114.42,"modelScoreStateId":"gemma-3-4b-it::default","modelScoreConfigSlug":"gemma-3-4b-it","modelScoreStateName":"Gemma 3 4B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":617.8811,"rankLo":590.0,"rankHi":644.0},{"slug":"qwen2-5-coder-7b-instruct","modelId":"qwen2-5-coder-7b-instruct","modelSlug":"qwen2-5-coder-7b-instruct","name":"Qwen2.5-Coder-7B-Instruct","organizationSlug":"alibaba","n":21,"rankEligible":true,"score":110.47,"scoreLo":105.07,"scoreHi":115.25,"modelScoreStateId":"qwen2-5-coder-7b-instruct::default","modelScoreConfigSlug":"qwen2-5-coder-7b-instruct","modelScoreStateName":"Qwen2.5-Coder-7B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":617.5469,"rankLo":575.0,"rankHi":658.0},{"slug":"qwen1-5-72b","modelId":"qwen1-5-72b","modelSlug":"qwen1-5-72b","name":"Qwen1.5 72B","organizationSlug":"alibaba","n":30,"rankEligible":true,"score":110.46,"scoreLo":105.63,"scoreHi":114.73,"modelScoreStateId":"qwen1-5-72b::default","modelScoreConfigSlug":"qwen1-5-72b","modelScoreStateName":"Qwen1.5 72B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":618.005,"rankLo":584.0,"rankHi":648.0},{"slug":"internvl3-2b","modelId":"internvl3-2b","modelSlug":"internvl3-2b","name":"InternVL3-2B","organizationSlug":"opengvlab","n":18,"rankEligible":true,"score":110.35,"scoreLo":105.03,"scoreHi":114.94,"modelScoreStateId":"internvl3-2b::default","modelScoreConfigSlug":"internvl3-2b","modelScoreStateName":"InternVL3-2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":619.1408,"rankLo":576.975,"rankHi":658.025},{"slug":"c4ai-command-r-plus","modelId":"c4ai-command-r-plus","modelSlug":"c4ai-command-r-plus","name":"C4AI Command R+","organizationSlug":"cohere","n":34,"rankEligible":true,"score":110.33,"scoreLo":105.49,"scoreHi":114.24,"modelScoreStateId":"c4ai-command-r-plus::default","modelScoreConfigSlug":"c4ai-command-r-plus","modelScoreStateName":"C4AI Command R+","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":619.5461,"rankLo":591.975,"rankHi":647.0},{"slug":"phi-4-multimodal-instruct","modelId":"phi-4-multimodal-instruct","modelSlug":"phi-4-multimodal-instruct","name":"Phi-4-multimodal-instruct","organizationSlug":"microsoft","n":22,"rankEligible":true,"score":110.33,"scoreLo":104.78,"scoreHi":115.05,"modelScoreStateId":"phi-4-multimodal-instruct::default","modelScoreConfigSlug":"phi-4-multimodal-instruct","modelScoreStateName":"Phi-4-multimodal-instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":619.1744,"rankLo":577.975,"rankHi":659.0},{"slug":"lfm3-omni-2b","modelId":"lfm3-omni-2b","modelSlug":"lfm3-omni-2b","name":"LFM3-Omni-2B","organizationSlug":"liquid-ai","n":12,"rankEligible":true,"score":110.19,"scoreLo":103.47,"scoreHi":116.0,"modelScoreStateId":"lfm3-omni-2b::default","modelScoreConfigSlug":"lfm3-omni-2b","modelScoreStateName":"LFM3-Omni-2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":620.3178,"rankLo":561.0,"rankHi":676.0},{"slug":"nemotron-h-8b-base-8k","modelId":"nemotron-h-8b-base-8k","modelSlug":"nemotron-h-8b-base-8k","name":"Nemotron-H 8B Base 8K","organizationSlug":"nvidia","n":12,"rankEligible":true,"score":110.09,"scoreLo":104.1,"scoreHi":115.57,"modelScoreStateId":"nemotron-h-8b-base-8k::default","modelScoreConfigSlug":"nemotron-h-8b-base-8k","modelScoreStateName":"Nemotron-H 8B Base 8K","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":621.7136,"rankLo":567.975,"rankHi":668.0},{"slug":"qwen2-5-coder-1-5b-instruct","modelId":"qwen2-5-coder-1-5b-instruct","modelSlug":"qwen2-5-coder-1-5b-instruct","name":"Qwen2.5 Coder 1.5B Instruct","organizationSlug":"alibaba","n":7,"rankEligible":true,"score":109.8,"scoreLo":99.36,"scoreHi":120.08,"modelScoreStateId":"qwen2-5-coder-1-5b-instruct::default","modelScoreConfigSlug":"qwen2-5-coder-1-5b-instruct","modelScoreStateName":"Qwen2.5 Coder 1.5B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":622.3794,"rankLo":508.0,"rankHi":716.0},{"slug":"claude-instant-1-2","modelId":"claude-instant-1-2","modelSlug":"claude-instant-1-2","name":"Claude Instant 1.2","organizationSlug":"anthropic","n":6,"rankEligible":true,"score":109.54,"scoreLo":102.97,"scoreHi":115.2,"modelScoreStateId":"claude-instant-1-2::default","modelScoreConfigSlug":"claude-instant-1-2","modelScoreStateName":"Claude Instant 1.2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":627.9581,"rankLo":571.0,"rankHi":683.0},{"slug":"mistral-nemo-instruct-2407","modelId":"mistral-nemo-instruct-2407","modelSlug":"mistral-nemo-instruct-2407","name":"Mistral NeMo Instruct 2407","organizationSlug":"mistral-ai","n":26,"rankEligible":true,"score":109.47,"scoreLo":104.23,"scoreHi":113.9,"modelScoreStateId":"mistral-nemo-instruct-2407::default","modelScoreConfigSlug":"mistral-nemo-instruct-2407","modelScoreStateName":"Mistral NeMo Instruct 2407","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":629.4025,"rankLo":597.0,"rankHi":661.0},{"slug":"dbrx-instruct","modelId":"dbrx-instruct","modelSlug":"dbrx-instruct","name":"DBRX Instruct","organizationSlug":"databricks","n":21,"rankEligible":true,"score":109.3,"scoreLo":104.52,"scoreHi":113.66,"modelScoreStateId":"dbrx-instruct::default","modelScoreConfigSlug":"dbrx-instruct","modelScoreStateName":"DBRX Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":631.4575,"rankLo":602.0,"rankHi":659.0},{"slug":"c4ai-command-r7b-12-2024","modelId":"c4ai-command-r7b-12-2024","modelSlug":"c4ai-command-r7b-12-2024","name":"Command R7B (Dec 2024)","organizationSlug":"cohere","n":13,"rankEligible":true,"score":109.2,"scoreLo":103.21,"scoreHi":114.67,"modelScoreStateId":"c4ai-command-r7b-12-2024::default","modelScoreConfigSlug":"c4ai-command-r7b-12-2024","modelScoreStateName":"Command R7B (Dec 2024)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":631.9994,"rankLo":581.0,"rankHi":679.0},{"slug":"gemma-3n-e4b-it","modelId":"gemma-3n-e4b-it","modelSlug":"gemma-3n-e4b-it","name":"Gemma 3n E4B Instruct","organizationSlug":"google","n":25,"rankEligible":true,"score":109.17,"scoreLo":103.83,"scoreHi":113.77,"modelScoreStateId":"gemma-3n-e4b-it::default","modelScoreConfigSlug":"gemma-3n-e4b-it","modelScoreStateName":"Gemma 3n E4B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":632.8608,"rankLo":593.975,"rankHi":669.0},{"slug":"gpt-3-5-turbo-0125","modelId":"gpt-3-5-turbo-0125","modelSlug":"gpt-3-5-turbo-0125","name":"GPT-3.5 Turbo 0125","organizationSlug":"openai","n":23,"rankEligible":true,"score":109.13,"scoreLo":103.54,"scoreHi":113.98,"modelScoreStateId":"gpt-3-5-turbo-0125::default","modelScoreConfigSlug":"gpt-3-5-turbo-0125","modelScoreStateName":"GPT-3.5 Turbo 0125","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":633.0453,"rankLo":595.0,"rankHi":670.0},{"slug":"gemini-pro","modelId":"gemini-pro","modelSlug":"gemini-pro","name":"Gemini 1.0 Pro","organizationSlug":"google","n":20,"rankEligible":true,"score":109.03,"scoreLo":103.97,"scoreHi":113.27,"modelScoreStateId":"gemini-pro::default","modelScoreConfigSlug":"gemini-pro","modelScoreStateName":"Gemini 1.0 Pro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":634.6358,"rankLo":606.0,"rankHi":662.0},{"slug":"palm-2-m","modelId":"palm-2-m","modelSlug":"palm-2-m","name":"PaLM 2-M","organizationSlug":"google","n":8,"rankEligible":true,"score":108.85,"scoreLo":100.07,"scoreHi":116.95,"modelScoreStateId":"palm-2-m::default","modelScoreConfigSlug":"palm-2-m","modelScoreStateName":"PaLM 2-M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":634.1947,"rankLo":546.0,"rankHi":705.025},{"slug":"yi-1-5-9b-chat","modelId":"yi-1-5-9b-chat","modelSlug":"yi-1-5-9b-chat","name":"Yi 1.5 9B Chat","organizationSlug":"01-ai","n":22,"rankEligible":true,"score":108.79,"scoreLo":103.76,"scoreHi":112.86,"modelScoreStateId":"yi-1-5-9b-chat::default","modelScoreConfigSlug":"yi-1-5-9b-chat","modelScoreStateName":"Yi 1.5 9B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":637.3389,"rankLo":612.0,"rankHi":663.0},{"slug":"moonlight-16b-a3b","modelId":"moonlight-16b-a3b","modelSlug":"moonlight-16b-a3b","name":"Moonlight 16B A3B","organizationSlug":"moonshot-ai","n":17,"rankEligible":true,"score":108.71,"scoreLo":103.03,"scoreHi":113.4,"modelScoreStateId":"moonlight-16b-a3b::default","modelScoreConfigSlug":"moonlight-16b-a3b","modelScoreStateName":"Moonlight 16B A3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":637.9672,"rankLo":601.0,"rankHi":673.0},{"slug":"lfm2-5-1-2b-instruct","modelId":"lfm2-5-1-2b-instruct","modelSlug":"lfm2-5-1-2b-instruct","name":"LFM2.5-1.2B-Instruct","organizationSlug":"liquid-ai","n":6,"rankEligible":true,"score":108.51,"scoreLo":93.57,"scoreHi":120.79,"modelScoreStateId":"lfm2-5-1-2b-instruct::default","modelScoreConfigSlug":"lfm2-5-1-2b-instruct","modelScoreStateName":"LFM2.5-1.2B-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":633.6219,"rankLo":498.0,"rankHi":764.0},{"slug":"deepseek-llm-67b-chat","modelId":"deepseek-llm-67b-chat","modelSlug":"deepseek-llm-67b-chat","name":"DeepSeek LLM 67B Chat","organizationSlug":"deepseek","n":43,"rankEligible":true,"score":108.42,"scoreLo":103.57,"scoreHi":112.64,"modelScoreStateId":"deepseek-llm-67b-chat::default","modelScoreConfigSlug":"deepseek-llm-67b-chat","modelScoreStateName":"DeepSeek LLM 67B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":641.4928,"rankLo":618.0,"rankHi":664.0},{"slug":"granite-4-0-micro","modelId":"granite-4-0-micro","modelSlug":"granite-4-0-micro","name":"Granite 4.0 Micro","organizationSlug":"ibm","n":15,"rankEligible":true,"score":108.4,"scoreLo":102.04,"scoreHi":113.92,"modelScoreStateId":"granite-4-0-micro::default","modelScoreConfigSlug":"granite-4-0-micro","modelScoreStateName":"Granite 4.0 Micro","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":640.8953,"rankLo":592.0,"rankHi":688.0},{"slug":"granite-3-3-8b-instruct","modelId":"granite-3-3-8b-instruct","modelSlug":"granite-3-3-8b-instruct","name":"Granite 3.3 8B Instruct","organizationSlug":"ibm","n":5,"rankEligible":true,"score":108.26,"scoreLo":101.23,"scoreHi":114.5,"modelScoreStateId":"granite-3-3-8b-instruct::default","modelScoreConfigSlug":"granite-3-3-8b-instruct-helm-capabilities","modelScoreStateName":"Granite 3.3 8B Instruct","modelScoreStateBestProbability":0.6641666666666667,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":642.0356,"rankLo":582.0,"rankHi":695.0},{"slug":"qwen2-7b-instruct","modelId":"qwen2-7b-instruct","modelSlug":"qwen2-7b-instruct","name":"Qwen2 7B Instruct","organizationSlug":"alibaba","n":20,"rankEligible":true,"score":108.26,"scoreLo":103.31,"scoreHi":112.56,"modelScoreStateId":"qwen2-7b-instruct::default","modelScoreConfigSlug":"qwen2-7b-instruct","modelScoreStateName":"Qwen2 7B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":643.1381,"rankLo":614.975,"rankHi":669.0},{"slug":"gemma-3-12b-pt","modelId":"gemma-3-12b-pt","modelSlug":"gemma-3-12b-pt","name":"Gemma 3 12B PT","organizationSlug":"google","n":22,"rankEligible":true,"score":108.24,"scoreLo":102.91,"scoreHi":112.8,"modelScoreStateId":"gemma-3-12b-pt::default","modelScoreConfigSlug":"gemma-3-12b-pt","modelScoreStateName":"Gemma 3 12B PT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":643.4017,"rankLo":613.0,"rankHi":674.0},{"slug":"internvl-chat-v1-5","modelId":"internvl-chat-v1-5","modelSlug":"internvl-chat-v1-5","name":"InternVL-Chat-V1-5","organizationSlug":"opengvlab","n":6,"rankEligible":true,"score":108.14,"scoreLo":101.09,"scoreHi":114.28,"modelScoreStateId":"internvl-chat-v1-5::default","modelScoreConfigSlug":"internvl-chat-v1-5","modelScoreStateName":"InternVL-Chat-V1-5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":643.3492,"rankLo":583.0,"rankHi":695.0},{"slug":"gpt-3-5-turbo-0613","modelId":"gpt-3-5-turbo-0613","modelSlug":"gpt-3-5-turbo-0613","name":"GPT-3.5 Turbo 0613","organizationSlug":"openai","n":7,"rankEligible":true,"score":108.05,"scoreLo":101.36,"scoreHi":114.1,"modelScoreStateId":"gpt-3-5-turbo-0613::default","modelScoreConfigSlug":"gpt-3-5-turbo-0613","modelScoreStateName":"GPT-3.5 Turbo 0613","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":644.5044,"rankLo":589.0,"rankHi":694.0},{"slug":"codestral-22b-v0-1","modelId":"codestral-22b-v0-1","modelSlug":"codestral-22b-v0-1","name":"Codestral 2405","organizationSlug":"mistral-ai","n":14,"rankEligible":true,"score":108.01,"scoreLo":102.16,"scoreHi":113.03,"modelScoreStateId":"codestral-22b-v0-1::default","modelScoreConfigSlug":"codestral-22b-v0-1","modelScoreStateName":"Codestral 2405","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":645.5219,"rankLo":604.0,"rankHi":684.0},{"slug":"medgemma-4b-it","modelId":"medgemma-4b-it","modelSlug":"medgemma-4b-it","name":"MedGemma 4B IT","organizationSlug":"google","n":7,"rankEligible":true,"score":107.81,"scoreLo":100.83,"scoreHi":114.11,"modelScoreStateId":"medgemma-4b-it::default","modelScoreConfigSlug":"medgemma-4b-it","modelScoreStateName":"MedGemma 4B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":646.8611,"rankLo":583.0,"rankHi":701.0},{"slug":"deepseek-llm-67b-base","modelId":"deepseek-llm-67b-base","modelSlug":"deepseek-llm-67b-base","name":"DeepSeek LLM 67B Base","organizationSlug":"deepseek","n":30,"rankEligible":true,"score":107.75,"scoreLo":102.37,"scoreHi":112.46,"modelScoreStateId":"deepseek-llm-67b-base::default","modelScoreConfigSlug":"deepseek-llm-67b-base","modelScoreStateName":"DeepSeek LLM 67B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":648.6197,"rankLo":618.0,"rankHi":677.0},{"slug":"tulu-2-dpo-70b","modelId":"tulu-2-dpo-70b","modelSlug":"tulu-2-dpo-70b","name":"T\u00fclu 2 DPO 70B","organizationSlug":"allen-ai","n":9,"rankEligible":true,"score":107.69,"scoreLo":101.74,"scoreHi":112.86,"modelScoreStateId":"tulu-2-dpo-70b::default","modelScoreConfigSlug":"tulu-2-dpo-70b","modelScoreStateName":"T\u00fclu 2 DPO 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":648.9456,"rankLo":606.975,"rankHi":686.0},{"slug":"phi-3-5-mini-instruct","modelId":"phi-3-5-mini-instruct","modelSlug":"phi-3-5-mini-instruct","name":"Phi-3.5 Mini Instruct","organizationSlug":"microsoft","n":31,"rankEligible":true,"score":107.63,"scoreLo":102.46,"scoreHi":111.92,"modelScoreStateId":"phi-3-5-mini-instruct::default","modelScoreConfigSlug":"phi-3-5-mini-instruct","modelScoreStateName":"Phi-3.5 Mini Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":650.0317,"rankLo":624.0,"rankHi":674.0},{"slug":"apertus-8b-instruct-2509","modelId":"apertus-8b-instruct-2509","modelSlug":"apertus-8b-instruct-2509","name":"Apertus 8B Instruct","organizationSlug":"swiss-ai","n":8,"rankEligible":true,"score":107.62,"scoreLo":94.16,"scoreHi":118.42,"modelScoreStateId":"apertus-8b-instruct-2509::default","modelScoreConfigSlug":"apertus-8b-instruct-2509","modelScoreStateName":"Apertus 8B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":644.23,"rankLo":529.0,"rankHi":760.0},{"slug":"llama-3-1-8b-instruct","modelId":"llama-3-1-8b-instruct","modelSlug":"llama-3-1-8b-instruct","name":"Llama 3.1 8B Instruct","organizationSlug":"meta","n":89,"rankEligible":true,"score":107.61,"scoreLo":102.86,"scoreHi":111.61,"modelScoreStateId":"llama-3-1-8b-instruct::default","modelScoreConfigSlug":"llama-3-1-8b-instruct","modelScoreStateName":"Llama 3.1 8B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":650.5833,"rankLo":631.0,"rankHi":669.0},{"slug":"deepseek-coder-v2-lite-instruct","modelId":"deepseek-coder-v2-lite-instruct","modelSlug":"deepseek-coder-v2-lite-instruct","name":"DeepSeek-Coder-V2-Lite-Instruct","organizationSlug":"deepseek","n":26,"rankEligible":true,"score":107.39,"scoreLo":102.24,"scoreHi":111.96,"modelScoreStateId":"deepseek-coder-v2-lite-instruct::default","modelScoreConfigSlug":"deepseek-coder-v2-lite-instruct","modelScoreStateName":"DeepSeek-Coder-V2-Lite-Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":652.6294,"rankLo":623.0,"rankHi":680.0},{"slug":"yi-34b-chat","modelId":"yi-34b-chat","modelSlug":"yi-34b-chat","name":"Yi 34B Chat","organizationSlug":"01-ai","n":23,"rankEligible":true,"score":107.35,"scoreLo":101.96,"scoreHi":111.71,"modelScoreStateId":"yi-34b-chat::default","modelScoreConfigSlug":"yi-34b-chat","modelScoreStateName":"Yi 34B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":653.1597,"rankLo":625.0,"rankHi":680.0},{"slug":"gemma-2-9b","modelId":"gemma-2-9b","modelSlug":"gemma-2-9b","name":"Gemma 2 9B","organizationSlug":"google","n":28,"rankEligible":true,"score":107.14,"scoreLo":101.9,"scoreHi":111.64,"modelScoreStateId":"gemma-2-9b::default","modelScoreConfigSlug":"gemma-2-9b","modelScoreStateName":"Gemma 2 9B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":655.375,"rankLo":629.0,"rankHi":680.0},{"slug":"phi-4-mini-instruct","modelId":"phi-4-mini-instruct","modelSlug":"phi-4-mini-instruct","name":"Phi-4-mini-instruct","organizationSlug":"microsoft","n":40,"rankEligible":true,"score":107.0,"scoreLo":101.71,"scoreHi":111.48,"modelScoreStateId":"phi-4-mini-instruct::default","modelScoreConfigSlug":"phi-4-mini-instruct","modelScoreStateName":"Phi-4-mini-instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":656.7906,"rankLo":630.0,"rankHi":682.0},{"slug":"ministral-3b-2410","modelId":"ministral-3b-2410","modelSlug":"ministral-3b-2410","name":"Ministral 3B","organizationSlug":"mistral-ai","n":5,"rankEligible":true,"score":106.81,"scoreLo":97.25,"scoreHi":115.23,"modelScoreStateId":"ministral-3b-2410::default","modelScoreConfigSlug":"ministral-3b-2410","modelScoreStateName":"Ministral 3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":655.4089,"rankLo":570.0,"rankHi":731.0},{"slug":"phi-3-mini-4k-instruct","modelId":"phi-3-mini-4k-instruct","modelSlug":"phi-3-mini-4k-instruct","name":"Phi-3-mini-4k-instruct","organizationSlug":"microsoft","n":25,"rankEligible":true,"score":106.8,"scoreLo":101.45,"scoreHi":111.4,"modelScoreStateId":"phi-3-mini-4k-instruct::default","modelScoreConfigSlug":"phi-3-mini-4k-instruct","modelScoreStateName":"Phi-3-mini-4k-instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":658.9042,"rankLo":632.0,"rankHi":683.0},{"slug":"llama-3-2-11b-vision-instruct","modelId":"llama-3-2-11b-vision-instruct","modelSlug":"llama-3-2-11b-vision-instruct","name":"Llama 3.2 11B Vision Instruct","organizationSlug":"meta","n":38,"rankEligible":true,"score":106.79,"scoreLo":101.49,"scoreHi":111.46,"modelScoreStateId":"llama-3-2-11b-vision-instruct::default","modelScoreConfigSlug":"llama-3-2-11b-vision-instruct","modelScoreStateName":"Llama 3.2 11B Vision Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":658.9675,"rankLo":629.0,"rankHi":686.0},{"slug":"gemini-pro-vision","modelId":"gemini-pro-vision","modelSlug":"gemini-pro-vision","name":"Gemini 1.0 Pro Vision","organizationSlug":"google","n":7,"rankEligible":true,"score":106.6,"scoreLo":98.54,"scoreHi":113.24,"modelScoreStateId":"gemini-pro-vision::default","modelScoreConfigSlug":"gemini-pro-vision","modelScoreStateName":"Gemini 1.0 Pro Vision","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":658.9736,"rankLo":593.0,"rankHi":718.0},{"slug":"qwen2-7b","modelId":"qwen2-7b","modelSlug":"qwen2-7b","name":"Qwen2-7B","organizationSlug":"alibaba","n":20,"rankEligible":true,"score":106.46,"scoreLo":100.99,"scoreHi":111.18,"modelScoreStateId":"qwen2-7b::default","modelScoreConfigSlug":"qwen2-7b","modelScoreStateName":"Qwen2-7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":662.2106,"rankLo":630.0,"rankHi":689.0},{"slug":"ministral-8b-instruct-2410","modelId":"ministral-8b-instruct-2410","modelSlug":"ministral-8b-instruct-2410","name":"Ministral 8B Instruct 2410","organizationSlug":"mistral-ai","n":14,"rankEligible":true,"score":106.35,"scoreLo":99.87,"scoreHi":111.7,"modelScoreStateId":"ministral-8b-instruct-2410::default","modelScoreConfigSlug":"ministral-8b-instruct-2410","modelScoreStateName":"Ministral 8B Instruct 2410","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":662.7719,"rankLo":620.0,"rankHi":701.0},{"slug":"moonlight-16b-a3b-instruct","modelId":"moonlight-16b-a3b-instruct","modelSlug":"moonlight-16b-a3b-instruct","name":"Moonlight 16B A3B Instruct","organizationSlug":"moonshot-ai","n":6,"rankEligible":true,"score":106.12,"scoreLo":98.94,"scoreHi":112.09,"modelScoreStateId":"moonlight-16b-a3b-instruct::default","modelScoreConfigSlug":"moonlight-16b-a3b-instruct","modelScoreStateName":"Moonlight 16B A3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":664.755,"rankLo":614.0,"rankHi":710.0},{"slug":"jamba-mini-1-7","modelId":"jamba-mini-1-7","modelSlug":"jamba-mini-1-7","name":"Jamba Mini 1.7","organizationSlug":"ai21-labs","n":8,"rankEligible":true,"score":106.05,"scoreLo":98.0,"scoreHi":113.42,"modelScoreStateId":"jamba-mini-1-7::default","modelScoreConfigSlug":"jamba-mini-1-7","modelScoreStateName":"Jamba Mini 1.7","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":664.3733,"rankLo":595.0,"rankHi":723.0},{"slug":"tiny-aya-global","modelId":"tiny-aya-global","modelSlug":"tiny-aya-global","name":"Tiny Aya Global","organizationSlug":"cohere","n":9,"rankEligible":true,"score":105.97,"scoreLo":95.11,"scoreHi":114.72,"modelScoreStateId":"tiny-aya-global::default","modelScoreConfigSlug":"tiny-aya-global","modelScoreStateName":"Tiny Aya Global","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":663.495,"rankLo":574.0,"rankHi":749.0},{"slug":"codellama-70b-python-hf","modelId":"codellama-70b-python-hf","modelSlug":"codellama-70b-python-hf","name":"Code Llama Python 70B","organizationSlug":"meta","n":9,"rankEligible":true,"score":105.89,"scoreLo":97.32,"scoreHi":113.54,"modelScoreStateId":"codellama-70b-python-hf::default","modelScoreConfigSlug":"codellama-70b-python-hf","modelScoreStateName":"Code Llama Python 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":665.5497,"rankLo":592.975,"rankHi":725.025},{"slug":"qwen2-5-coder-7b","modelId":"qwen2-5-coder-7b","modelSlug":"qwen2-5-coder-7b","name":"Qwen2.5 Coder 7B","organizationSlug":"alibaba","n":20,"rankEligible":true,"score":105.86,"scoreLo":100.1,"scoreHi":110.74,"modelScoreStateId":"qwen2-5-coder-7b::default","modelScoreConfigSlug":"qwen2-5-coder-7b","modelScoreStateName":"Qwen2.5 Coder 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":668.1769,"rankLo":637.0,"rankHi":696.0},{"slug":"phi-3-mini-128k-instruct","modelId":"phi-3-mini-128k-instruct","modelSlug":"phi-3-mini-128k-instruct","name":"Phi-3 Mini 128K Instruct","organizationSlug":"microsoft","n":15,"rankEligible":true,"score":105.83,"scoreLo":99.75,"scoreHi":110.92,"modelScoreStateId":"phi-3-mini-128k-instruct::default","modelScoreConfigSlug":"phi-3-mini-128k-instruct","modelScoreStateName":"Phi-3 Mini 128K Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":668.2611,"rankLo":635.0,"rankHi":699.0},{"slug":"deepseek-r1-distill-llama-8b","modelId":"deepseek-r1-distill-llama-8b","modelSlug":"deepseek-r1-distill-llama-8b","name":"DeepSeek R1 Distill Llama 8B","organizationSlug":"deepseek","n":21,"rankEligible":true,"score":105.76,"scoreLo":99.84,"scoreHi":110.87,"modelScoreStateId":"deepseek-r1-distill-llama-8b::default","modelScoreConfigSlug":"deepseek-r1-distill-llama-8b","modelScoreStateName":"DeepSeek R1 Distill Llama 8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":668.9175,"rankLo":634.0,"rankHi":703.0},{"slug":"grok-1-2023-11","modelId":"grok-1-2023-11","modelSlug":"grok-1-2023-11","name":"Grok 1 (November 2023)","organizationSlug":"xai","n":4,"rankEligible":true,"score":105.68,"scoreLo":96.86,"scoreHi":113.79,"modelScoreStateId":"grok-1-2023-11::default","modelScoreConfigSlug":"grok-1-2023-11","modelScoreStateName":"Grok 1 (November 2023)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":667.3894,"rankLo":590.975,"rankHi":727.0},{"slug":"ministral-3-3b-base-2512","modelId":"ministral-3-3b-base-2512","modelSlug":"ministral-3-3b-base-2512","name":"Ministral 3 3B Base 2512","organizationSlug":"mistral-ai","n":10,"rankEligible":true,"score":105.67,"scoreLo":99.57,"scoreHi":111.01,"modelScoreStateId":"ministral-3-3b-base-2512::default","modelScoreConfigSlug":"ministral-3-3b-base-2512","modelScoreStateName":"Ministral 3 3B Base 2512","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":669.5703,"rankLo":630.0,"rankHi":705.0},{"slug":"gpt-3-5-turbo-1106","modelId":"gpt-3-5-turbo-1106","modelSlug":"gpt-3-5-turbo-1106","name":"GPT-3.5 Turbo 1106","organizationSlug":"openai","n":6,"rankEligible":true,"score":105.66,"scoreLo":99.37,"scoreHi":111.14,"modelScoreStateId":"gpt-3-5-turbo-1106::default","modelScoreConfigSlug":"gpt-3-5-turbo-1106","modelScoreStateName":"GPT-3.5 Turbo 1106","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":669.7022,"rankLo":630.0,"rankHi":707.0},{"slug":"nous-hermes-2-mixtral-8x7b-sft","modelId":"nous-hermes-2-mixtral-8x7b-sft","modelSlug":"nous-hermes-2-mixtral-8x7b-sft","name":"Nous Hermes 2 Mixtral 8x7B SFT","organizationSlug":"nous-research","n":13,"rankEligible":true,"score":105.6,"scoreLo":99.95,"scoreHi":110.66,"modelScoreStateId":"nous-hermes-2-mixtral-8x7b-sft::default","modelScoreConfigSlug":"nous-hermes-2-mixtral-8x7b-sft","modelScoreStateName":"Nous Hermes 2 Mixtral 8x7B SFT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":670.6542,"rankLo":637.0,"rankHi":700.0},{"slug":"mixtral-8x7b-v0-1","modelId":"mixtral-8x7b-v0-1","modelSlug":"mixtral-8x7b-v0-1","name":"Mixtral 8x7B","organizationSlug":"mistral-ai","n":27,"rankEligible":true,"score":105.56,"scoreLo":100.11,"scoreHi":110.22,"modelScoreStateId":"mixtral-8x7b-v0-1::default","modelScoreConfigSlug":"mixtral-8x7b-v0-1","modelScoreStateName":"Mixtral 8x7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":671.2286,"rankLo":646.0,"rankHi":694.0},{"slug":"mixtral-8x7b-instruct-v0-1","modelId":"mixtral-8x7b-instruct-v0-1","modelSlug":"mixtral-8x7b-instruct-v0-1","name":"Mixtral 8x7B Instruct","organizationSlug":"mistral-ai","n":37,"rankEligible":true,"score":105.5,"scoreLo":100.2,"scoreHi":109.92,"modelScoreStateId":"mixtral-8x7b-instruct-v0-1::default","modelScoreConfigSlug":"mixtral-8x7b-instruct-v0-1","modelScoreStateName":"Mixtral 8x7B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":671.9961,"rankLo":650.0,"rankHi":692.0},{"slug":"reka-flash-20240226","modelId":"reka-flash-20240226","modelSlug":"reka-flash-20240226","name":"Reka Flash (2024-02-26)","organizationSlug":"reka-ai","n":5,"rankEligible":true,"score":105.18,"scoreLo":98.47,"scoreHi":110.86,"modelScoreStateId":"reka-flash-20240226::default","modelScoreConfigSlug":"reka-flash-20240226","modelScoreStateName":"Reka Flash (2024-02-26)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":674.1897,"rankLo":630.0,"rankHi":714.0},{"slug":"qwen2-5-coder-3b-instruct","modelId":"qwen2-5-coder-3b-instruct","modelSlug":"qwen2-5-coder-3b-instruct","name":"Qwen2.5 Coder 3B Instruct","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":104.85,"scoreLo":97.99,"scoreHi":111.04,"modelScoreStateId":"qwen2-5-coder-3b-instruct::default","modelScoreConfigSlug":"qwen2-5-coder-3b-instruct","modelScoreStateName":"Qwen2.5 Coder 3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":676.9089,"rankLo":631.0,"rankHi":717.0},{"slug":"qwen2-5-3b-instruct","modelId":"qwen2-5-3b-instruct","modelSlug":"qwen2-5-3b-instruct","name":"Qwen2.5 3B Instruct","organizationSlug":"alibaba","n":38,"rankEligible":true,"score":104.73,"scoreLo":99.56,"scoreHi":109.49,"modelScoreStateId":"qwen2-5-3b-instruct::default","modelScoreConfigSlug":"qwen2-5-3b-instruct","modelScoreStateName":"Qwen2.5 3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":679.2053,"rankLo":658.0,"rankHi":699.0},{"slug":"jamba-mini-1-5","modelId":"jamba-mini-1-5","modelSlug":"jamba-mini-1-5","name":"Jamba 1.5 Mini","organizationSlug":"ai21-labs","n":7,"rankEligible":true,"score":104.65,"scoreLo":97.75,"scoreHi":110.73,"modelScoreStateId":"jamba-mini-1-5::default","modelScoreConfigSlug":"jamba-mini-1-5","modelScoreStateName":"Jamba 1.5 Mini","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":678.8394,"rankLo":631.975,"rankHi":720.0},{"slug":"qwen1-5-14b-chat","modelId":"qwen1-5-14b-chat","modelSlug":"qwen1-5-14b-chat","name":"Qwen1.5 14B Chat","organizationSlug":"alibaba","n":17,"rankEligible":true,"score":104.62,"scoreLo":98.87,"scoreHi":109.58,"modelScoreStateId":"qwen1-5-14b-chat::default","modelScoreConfigSlug":"qwen1-5-14b-chat","modelScoreStateName":"Qwen1.5 14B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":679.9928,"rankLo":654.0,"rankHi":705.0},{"slug":"quyen-pro-v0-1","modelId":"quyen-pro-v0-1","modelSlug":"quyen-pro-v0-1","name":"Quyen Pro V0.1","organizationSlug":"vilm","n":7,"rankEligible":true,"score":104.47,"scoreLo":97.99,"scoreHi":110.32,"modelScoreStateId":"quyen-pro-v0-1::default","modelScoreConfigSlug":"quyen-pro-v0-1","modelScoreStateName":"Quyen Pro V0.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":680.7097,"rankLo":637.0,"rankHi":718.0},{"slug":"stablebeluga2","modelId":"stablebeluga2","modelSlug":"stablebeluga2","name":"Stable Beluga 2","organizationSlug":"stability-ai","n":23,"rankEligible":true,"score":104.46,"scoreLo":98.76,"scoreHi":109.33,"modelScoreStateId":"stablebeluga2::default","modelScoreConfigSlug":"stablebeluga2","modelScoreStateName":"Stable Beluga 2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":681.35,"rankLo":654.0,"rankHi":707.0},{"slug":"llama-2-70b-hf","modelId":"llama-2-70b-hf","modelSlug":"llama-2-70b-hf","name":"Llama 2 70B","organizationSlug":"meta","n":34,"rankEligible":true,"score":104.25,"scoreLo":98.59,"scoreHi":109.1,"modelScoreStateId":"llama-2-70b-hf::default","modelScoreConfigSlug":"llama-2-70b-hf","modelScoreStateName":"Llama 2 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":683.3,"rankLo":659.0,"rankHi":706.0},{"slug":"qwen2-vl-2b-instruct","modelId":"qwen2-vl-2b-instruct","modelSlug":"qwen2-vl-2b-instruct","name":"Qwen2 VL 2B Instruct","organizationSlug":"alibaba","n":21,"rankEligible":true,"score":104.15,"scoreLo":97.78,"scoreHi":109.58,"modelScoreStateId":"qwen2-vl-2b-instruct::default","modelScoreConfigSlug":"qwen2-vl-2b-instruct","modelScoreStateName":"Qwen2 VL 2B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":683.8653,"rankLo":648.975,"rankHi":716.0},{"slug":"chinchilla","modelId":"chinchilla","modelSlug":"chinchilla","name":"Chinchilla","organizationSlug":"google","n":8,"rankEligible":true,"score":103.95,"scoreLo":96.17,"scoreHi":111.03,"modelScoreStateId":"chinchilla::default","modelScoreConfigSlug":"chinchilla","modelScoreStateName":"Chinchilla","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":684.6092,"rankLo":624.0,"rankHi":735.0},{"slug":"yi-9b","modelId":"yi-9b","modelSlug":"yi-9b","name":"Yi-9B","organizationSlug":"01-ai","n":11,"rankEligible":true,"score":103.92,"scoreLo":97.8,"scoreHi":109.32,"modelScoreStateId":"yi-9b::default","modelScoreConfigSlug":"yi-9b","modelScoreStateName":"Yi-9B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":685.9678,"rankLo":650.975,"rankHi":718.0},{"slug":"qwen1-5-14b","modelId":"qwen1-5-14b","modelSlug":"qwen1-5-14b","name":"Qwen1.5 14B","organizationSlug":"alibaba","n":20,"rankEligible":true,"score":103.78,"scoreLo":97.87,"scoreHi":108.9,"modelScoreStateId":"qwen1-5-14b::default","modelScoreConfigSlug":"qwen1-5-14b","modelScoreStateName":"Qwen1.5 14B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":687.4189,"rankLo":660.0,"rankHi":713.0},{"slug":"llama-3-8b-instruct","modelId":"llama-3-8b-instruct","modelSlug":"llama-3-8b-instruct","name":"Llama 3 8B Instruct","organizationSlug":"meta","n":56,"rankEligible":true,"score":103.65,"scoreLo":98.36,"scoreHi":108.26,"modelScoreStateId":"llama-3-8b-instruct::default","modelScoreConfigSlug":"llama-3-8b-instruct","modelScoreStateName":"Llama 3 8B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":688.8244,"rankLo":672.975,"rankHi":703.0},{"slug":"aya-23-35b","modelId":"aya-23-35b","modelSlug":"aya-23-35b","name":"Aya-23-35B","organizationSlug":"cohere","n":8,"rankEligible":true,"score":103.6,"scoreLo":96.06,"scoreHi":110.06,"modelScoreStateId":"aya-23-35b::default","modelScoreConfigSlug":"aya-23-35b","modelScoreStateName":"Aya-23-35B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":688.1925,"rankLo":643.0,"rankHi":730.0},{"slug":"qwen3-5-0-8b","modelId":"qwen3-5-0-8b","modelSlug":"qwen3-5-0-8b","name":"Qwen3.5-0.8B","organizationSlug":"alibaba","n":54,"rankEligible":true,"score":103.47,"scoreLo":97.75,"scoreHi":108.23,"modelScoreStateId":"qwen3-5-0-8b::enabled","modelScoreConfigSlug":"qwen3-5-0-8b-thinking","modelScoreStateName":"Qwen3.5-0.8B (thinking)","modelScoreStateBestProbability":0.9897222222222222,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":690.3808,"rankLo":669.0,"rankHi":710.0},{"slug":"lfm2-5-vl-1-6b","modelId":"lfm2-5-vl-1-6b","modelSlug":"lfm2-5-vl-1-6b","name":"LFM2.5-VL-1.6B","organizationSlug":"liquid-ai","n":6,"rankEligible":true,"score":103.41,"scoreLo":91.6,"scoreHi":112.91,"modelScoreStateId":"lfm2-5-vl-1-6b::default","modelScoreConfigSlug":"lfm2-5-vl-1-6b","modelScoreStateName":"LFM2.5-VL-1.6B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":687.4628,"rankLo":598.975,"rankHi":771.0},{"slug":"codellama-70b-instruct-hf","modelId":"codellama-70b-instruct-hf","modelSlug":"codellama-70b-instruct-hf","name":"Code Llama Instruct 70B","organizationSlug":"meta","n":12,"rankEligible":true,"score":102.89,"scoreLo":95.64,"scoreHi":109.03,"modelScoreStateId":"codellama-70b-instruct-hf::default","modelScoreConfigSlug":"codellama-70b-instruct","modelScoreStateName":"Code Llama Instruct 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":694.4317,"rankLo":651.0,"rankHi":733.0},{"slug":"codellama-70b-hf","modelId":"codellama-70b-hf","modelSlug":"codellama-70b-hf","name":"Code Llama 70B","organizationSlug":"meta","n":10,"rankEligible":true,"score":102.77,"scoreLo":95.02,"scoreHi":109.62,"modelScoreStateId":"codellama-70b-hf::default","modelScoreConfigSlug":"codellama-70b-hf","modelScoreStateName":"Code Llama 70B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":695.3378,"rankLo":645.0,"rankHi":738.0},{"slug":"palm-2-s","modelId":"palm-2-s","modelSlug":"palm-2-s","name":"PaLM 2-S","organizationSlug":"google","n":8,"rankEligible":true,"score":102.62,"scoreLo":93.72,"scoreHi":110.85,"modelScoreStateId":"palm-2-s::default","modelScoreConfigSlug":"palm-2-s","modelScoreStateName":"PaLM 2-S","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":695.79,"rankLo":625.0,"rankHi":753.0},{"slug":"deepseek-vl2-tiny","modelId":"deepseek-vl2-tiny","modelSlug":"deepseek-vl2-tiny","name":"DeepSeek-VL2-Tiny","organizationSlug":"deepseek","n":7,"rankEligible":true,"score":102.58,"scoreLo":95.06,"scoreHi":109.2,"modelScoreStateId":"deepseek-vl2-tiny::default","modelScoreConfigSlug":"deepseek-vl2-tiny","modelScoreStateName":"DeepSeek-VL2-Tiny","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":697.0636,"rankLo":649.0,"rankHi":742.0},{"slug":"inflection-1","modelId":"inflection-1","modelSlug":"inflection-1","name":"Inflection-1","organizationSlug":"inflection-ai","n":4,"rankEligible":true,"score":102.56,"scoreLo":93.64,"scoreHi":110.5,"modelScoreStateId":"inflection-1::default","modelScoreConfigSlug":"inflection-1","modelScoreStateName":"Inflection-1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":696.4483,"rankLo":631.0,"rankHi":753.0},{"slug":"c4ai-command-r-v01","modelId":"c4ai-command-r-v01","modelSlug":"c4ai-command-r-v01","name":"Command R","organizationSlug":"cohere","n":26,"rankEligible":true,"score":102.54,"scoreLo":96.53,"scoreHi":107.47,"modelScoreStateId":"c4ai-command-r-v01::default","modelScoreConfigSlug":"c4ai-command-r-v01","modelScoreStateName":"Command R","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":698.2225,"rankLo":675.0,"rankHi":720.0},{"slug":"llama-3-2-3b-instruct","modelId":"llama-3-2-3b-instruct","modelSlug":"llama-3-2-3b-instruct","name":"Llama 3.2 3B Instruct","organizationSlug":"meta","n":43,"rankEligible":true,"score":102.52,"scoreLo":96.69,"scoreHi":107.4,"modelScoreStateId":"llama-3-2-3b-instruct::default","modelScoreConfigSlug":"llama-3-2-3b-instruct","modelScoreStateName":"Llama 3.2 3B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":698.3542,"rankLo":678.0,"rankHi":718.0},{"slug":"gpt-3-5-turbo-0301","modelId":"gpt-3-5-turbo-0301","modelSlug":"gpt-3-5-turbo-0301","name":"GPT-3.5 Turbo 0301","organizationSlug":"openai","n":5,"rankEligible":true,"score":102.44,"scoreLo":93.71,"scoreHi":109.55,"modelScoreStateId":"gpt-3-5-turbo-0301::default","modelScoreConfigSlug":"gpt-3-5-turbo-0301","modelScoreStateName":"GPT-3.5 Turbo 0301","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":697.8956,"rankLo":641.0,"rankHi":754.025},{"slug":"qwen3-1-7b-base","modelId":"qwen3-1-7b-base","modelSlug":"qwen3-1-7b-base","name":"Qwen3 1.7B Base","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":101.87,"scoreLo":95.46,"scoreHi":107.34,"modelScoreStateId":"qwen3-1-7b-base::default","modelScoreConfigSlug":"qwen3-1-7b-base","modelScoreStateName":"Qwen3 1.7B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":703.5806,"rankLo":675.0,"rankHi":730.0},{"slug":"olmo-2-1124-13b-instruct","modelId":"olmo-2-1124-13b-instruct","modelSlug":"olmo-2-1124-13b-instruct","name":"OLMo 2 1124 13B Instruct","organizationSlug":"allen-ai","n":5,"rankEligible":true,"score":101.79,"scoreLo":93.66,"scoreHi":109.08,"modelScoreStateId":"olmo-2-1124-13b-instruct::default","modelScoreConfigSlug":"olmo-2-1124-13b-instruct-helm-capabilities","modelScoreStateName":"OLMo 2 1124 13B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":703.4233,"rankLo":649.975,"rankHi":756.0},{"slug":"yi-1-5-6b-chat","modelId":"yi-1-5-6b-chat","modelSlug":"yi-1-5-6b-chat","name":"Yi 1.5 6B Chat","organizationSlug":"01-ai","n":16,"rankEligible":true,"score":101.79,"scoreLo":95.2,"scoreHi":107.32,"modelScoreStateId":"yi-1-5-6b-chat::default","modelScoreConfigSlug":"yi-1-5-6b-chat","modelScoreStateName":"Yi 1.5 6B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":704.1222,"rankLo":675.0,"rankHi":730.0},{"slug":"llama-3-8b","modelId":"llama-3-8b","modelSlug":"llama-3-8b","name":"Llama 3 8B","organizationSlug":"meta","n":42,"rankEligible":true,"score":101.72,"scoreLo":96.05,"scoreHi":106.46,"modelScoreStateId":"llama-3-8b::default","modelScoreConfigSlug":"llama-3-8b","modelScoreStateName":"Llama 3 8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":705.1475,"rankLo":691.0,"rankHi":719.0},{"slug":"molmo-7b-d-0924","modelId":"molmo-7b-d-0924","modelSlug":"molmo-7b-d-0924","name":"Molmo 7B-D","organizationSlug":"allen-ai","n":14,"rankEligible":true,"score":101.72,"scoreLo":94.99,"scoreHi":107.56,"modelScoreStateId":"molmo-7b-d-0924::default","modelScoreConfigSlug":"molmo-7b-d-0924","modelScoreStateName":"Molmo 7B-D","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":704.6778,"rankLo":672.0,"rankHi":737.0},{"slug":"palm-540b","modelId":"palm-540b","modelSlug":"palm-540b","name":"PaLM 540B","organizationSlug":"google","n":16,"rankEligible":true,"score":101.71,"scoreLo":94.78,"scoreHi":107.8,"modelScoreStateId":"palm-540b::default","modelScoreConfigSlug":"palm-540b","modelScoreStateName":"PaLM 540B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":704.6653,"rankLo":668.0,"rankHi":738.0},{"slug":"openchat-3-5-1210","modelId":"openchat-3-5-1210","modelSlug":"openchat-3-5-1210","name":"OpenChat 3.5 1210","organizationSlug":"openchat","n":16,"rankEligible":true,"score":101.57,"scoreLo":94.95,"scoreHi":107.0,"modelScoreStateId":"openchat-3-5-1210::default","modelScoreConfigSlug":"openchat-3-5-1210","modelScoreStateName":"OpenChat 3.5 1210","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":705.9856,"rankLo":679.0,"rankHi":730.0},{"slug":"mistral-nemo-base-2407","modelId":"mistral-nemo-base-2407","modelSlug":"mistral-nemo-base-2407","name":"Mistral NeMo Base 2407","organizationSlug":"mistral-ai","n":15,"rankEligible":true,"score":101.54,"scoreLo":94.93,"scoreHi":107.37,"modelScoreStateId":"mistral-nemo-base-2407::default","modelScoreConfigSlug":"mistral-nemo-base-2407","modelScoreStateName":"Mistral NeMo Base 2407","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":706.1744,"rankLo":673.0,"rankHi":738.0},{"slug":"jamba-mini-1-6","modelId":"jamba-mini-1-6","modelSlug":"jamba-mini-1-6","name":"Jamba Mini 1.6","organizationSlug":"ai21-labs","n":9,"rankEligible":true,"score":101.45,"scoreLo":94.12,"scoreHi":107.69,"modelScoreStateId":"jamba-mini-1-6::default","modelScoreConfigSlug":"jamba-mini-1-6","modelScoreStateName":"Jamba Mini 1.6","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":706.7056,"rankLo":668.0,"rankHi":746.0},{"slug":"qwen2-5-coder-0-5b-instruct","modelId":"qwen2-5-coder-0-5b-instruct","modelSlug":"qwen2-5-coder-0-5b-instruct","name":"Qwen2.5 Coder 0.5B Instruct","organizationSlug":"alibaba","n":6,"rankEligible":true,"score":101.43,"scoreLo":89.38,"scoreHi":112.52,"modelScoreStateId":"qwen2-5-coder-0-5b-instruct::default","modelScoreConfigSlug":"qwen2-5-coder-0-5b-instruct","modelScoreStateName":"Qwen2.5 Coder 0.5B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":703.9411,"rankLo":594.0,"rankHi":784.0},{"slug":"internvl3-1b","modelId":"internvl3-1b","modelSlug":"internvl3-1b","name":"InternVL3-1B","organizationSlug":"opengvlab","n":7,"rankEligible":true,"score":101.22,"scoreLo":93.3,"scoreHi":107.96,"modelScoreStateId":"internvl3-1b::default","modelScoreConfigSlug":"internvl3-1b","modelScoreStateName":"InternVL3-1B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":708.4839,"rankLo":663.0,"rankHi":754.0},{"slug":"mistral-nemo-minitron-8b-base","modelId":"mistral-nemo-minitron-8b-base","modelSlug":"mistral-nemo-minitron-8b-base","name":"Mistral-NeMo-Minitron 8B Base","organizationSlug":"nvidia","n":7,"rankEligible":true,"score":101.06,"scoreLo":92.83,"scoreHi":108.37,"modelScoreStateId":"mistral-nemo-minitron-8b-base::default","modelScoreConfigSlug":"mistral-nemo-minitron-8b-base","modelScoreStateName":"Mistral-NeMo-Minitron 8B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":709.6014,"rankLo":658.0,"rankHi":757.0},{"slug":"qwen2-5-3b","modelId":"qwen2-5-3b","modelSlug":"qwen2-5-3b","name":"Qwen2.5 3B","organizationSlug":"alibaba","n":18,"rankEligible":true,"score":100.89,"scoreLo":94.52,"scoreHi":106.55,"modelScoreStateId":"qwen2-5-3b::default","modelScoreConfigSlug":"qwen2-5-3b","modelScoreStateName":"Qwen2.5 3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":711.4572,"rankLo":683.0,"rankHi":739.0},{"slug":"solar-10-7b-instruct-v1-0","modelId":"solar-10-7b-instruct-v1-0","modelSlug":"solar-10-7b-instruct-v1-0","name":"Solar 10.7B Instruct V1.0","organizationSlug":"upstage","n":27,"rankEligible":true,"score":100.88,"scoreLo":95.05,"scoreHi":105.74,"modelScoreStateId":"solar-10-7b-instruct-v1-0::default","modelScoreConfigSlug":"solar-10-7b-instruct-v1-0","modelScoreStateName":"Solar 10.7B Instruct V1.0","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":711.7594,"rankLo":697.0,"rankHi":725.0},{"slug":"llama-3-1-tulu-3-8b","modelId":"llama-3-1-tulu-3-8b","modelSlug":"llama-3-1-tulu-3-8b","name":"Llama-3.1-T\u00fclu-3-8B","organizationSlug":"allen-ai","n":7,"rankEligible":true,"score":100.87,"scoreLo":93.23,"scoreHi":107.49,"modelScoreStateId":"llama-3-1-tulu-3-8b::default","modelScoreConfigSlug":"llama-3-1-tulu-3-8b","modelScoreStateName":"Llama-3.1-T\u00fclu-3-8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":711.3858,"rankLo":669.0,"rankHi":756.0},{"slug":"qwen-14b","modelId":"qwen-14b","modelSlug":"qwen-14b","name":"Qwen 14B","organizationSlug":"alibaba","n":17,"rankEligible":true,"score":100.79,"scoreLo":94.19,"scoreHi":106.36,"modelScoreStateId":"qwen-14b::default","modelScoreConfigSlug":"qwen-14b","modelScoreStateName":"Qwen 14B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":712.2217,"rankLo":685.0,"rankHi":739.0},{"slug":"nemotron-h-4b-base-8k","modelId":"nemotron-h-4b-base-8k","modelSlug":"nemotron-h-4b-base-8k","name":"Nemotron-H 4B Base 8K","organizationSlug":"nvidia","n":10,"rankEligible":true,"score":100.4,"scoreLo":91.99,"scoreHi":107.4,"modelScoreStateId":"nemotron-h-4b-base-8k::default","modelScoreConfigSlug":"nemotron-h-4b-base-8k","modelScoreStateName":"Nemotron-H 4B Base 8K","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":715.1322,"rankLo":669.975,"rankHi":758.0},{"slug":"qwen-14b-chat","modelId":"qwen-14b-chat","modelSlug":"qwen-14b-chat","name":"Qwen 14B Chat","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":100.24,"scoreLo":93.25,"scoreHi":106.33,"modelScoreStateId":"qwen-14b-chat::default","modelScoreConfigSlug":"qwen-14b-chat","modelScoreStateName":"Qwen 14B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":716.6028,"rankLo":684.0,"rankHi":751.0},{"slug":"qwen2-5-coder-3b","modelId":"qwen2-5-coder-3b","modelSlug":"qwen2-5-coder-3b","name":"Qwen2.5-Coder-3B","organizationSlug":"alibaba","n":14,"rankEligible":true,"score":100.23,"scoreLo":93.25,"scoreHi":106.23,"modelScoreStateId":"qwen2-5-coder-3b::default","modelScoreConfigSlug":"qwen2-5-coder-3b","modelScoreStateName":"Qwen2.5-Coder-3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":716.6114,"rankLo":685.0,"rankHi":750.0},{"slug":"snorkel-mistral-pairrm-dpo","modelId":"snorkel-mistral-pairrm-dpo","modelSlug":"snorkel-mistral-pairrm-dpo","name":"Snorkel-Mistral-PairRM-DPO","organizationSlug":"snorkel-ai","n":7,"rankEligible":true,"score":99.92,"scoreLo":92.57,"scoreHi":106.13,"modelScoreStateId":"snorkel-mistral-pairrm-dpo::default","modelScoreConfigSlug":"snorkel-mistral-pairrm-dpo","modelScoreStateName":"Snorkel-Mistral-PairRM-DPO","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":719.0761,"rankLo":682.0,"rankHi":756.0},{"slug":"llama-3-1-8b","modelId":"llama-3-1-8b","modelSlug":"llama-3-1-8b","name":"Llama 3.1 8B","organizationSlug":"meta","n":28,"rankEligible":true,"score":99.81,"scoreLo":93.42,"scoreHi":105.42,"modelScoreStateId":"llama-3-1-8b::default","modelScoreConfigSlug":"llama-3-1-8b","modelScoreStateName":"Llama 3.1 8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":719.9322,"rankLo":696.0,"rankHi":745.0},{"slug":"neural-chat-7b-v3-1","modelId":"neural-chat-7b-v3-1","modelSlug":"neural-chat-7b-v3-1","name":"Neural Chat 7B v3-1","organizationSlug":"intel","n":12,"rankEligible":true,"score":99.78,"scoreLo":92.77,"scoreHi":105.62,"modelScoreStateId":"neural-chat-7b-v3-1::default","modelScoreConfigSlug":"neural-chat-7b-v3-1","modelScoreStateName":"Neural Chat 7B v3-1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":720.2461,"rankLo":692.0,"rankHi":749.025},{"slug":"granite-4-0-h-1b","modelId":"granite-4-0-h-1b","modelSlug":"granite-4-0-h-1b","name":"Granite 4.0 H 1B","organizationSlug":"ibm","n":7,"rankEligible":true,"score":99.45,"scoreLo":89.55,"scoreHi":107.88,"modelScoreStateId":"granite-4-0-h-1b::default","modelScoreConfigSlug":"granite-4-0-h-1b","modelScoreStateName":"Granite 4.0 H 1B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":722.2711,"rankLo":664.0,"rankHi":780.0},{"slug":"granite-4-0-1b","modelId":"granite-4-0-1b","modelSlug":"granite-4-0-1b","name":"Granite 4.0 1B","organizationSlug":"ibm","n":10,"rankEligible":true,"score":99.4,"scoreLo":90.3,"scoreHi":106.95,"modelScoreStateId":"granite-4-0-1b::default","modelScoreConfigSlug":"granite-4-0-1b","modelScoreStateName":"Granite 4.0 1B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":723.0094,"rankLo":672.0,"rankHi":774.0},{"slug":"gemma-3n-e2b-it","modelId":"gemma-3n-e2b-it","modelSlug":"gemma-3n-e2b-it","name":"Gemma 3n E2B Instruct","organizationSlug":"google","n":13,"rankEligible":true,"score":99.24,"scoreLo":91.26,"scoreHi":105.94,"modelScoreStateId":"gemma-3n-e2b-it::default","modelScoreConfigSlug":"gemma-3n-e2b-it","modelScoreStateName":"Gemma 3n E2B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":724.5833,"rankLo":683.0,"rankHi":768.0},{"slug":"wizardlm-2-7b","modelId":"wizardlm-2-7b","modelSlug":"wizardlm-2-7b","name":"WizardLM-2 7B","organizationSlug":"microsoft","n":7,"rankEligible":true,"score":99.22,"scoreLo":91.71,"scoreHi":105.78,"modelScoreStateId":"wizardlm-2-7b::default","modelScoreConfigSlug":"wizardlm-2-7b","modelScoreStateName":"WizardLM-2 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":724.6336,"rankLo":687.0,"rankHi":762.0},{"slug":"deepseek-coder-v2-lite-base","modelId":"deepseek-coder-v2-lite-base","modelSlug":"deepseek-coder-v2-lite-base","name":"DeepSeek-Coder-V2-Lite-Base","organizationSlug":"deepseek","n":14,"rankEligible":true,"score":98.92,"scoreLo":91.89,"scoreHi":105.1,"modelScoreStateId":"deepseek-coder-v2-lite-base::default","modelScoreConfigSlug":"deepseek-coder-v2-lite-base","modelScoreStateName":"DeepSeek-Coder-V2-Lite-Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":727.1172,"rankLo":696.0,"rankHi":761.0},{"slug":"gemma-7b","modelId":"gemma-7b","modelSlug":"gemma-7b","name":"Gemma 7B","organizationSlug":"google","n":42,"rankEligible":true,"score":98.51,"scoreLo":92.4,"scoreHi":103.63,"modelScoreStateId":"gemma-7b::default","modelScoreConfigSlug":"gemma-7b","modelScoreStateName":"Gemma 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":729.8108,"rankLo":716.975,"rankHi":743.025},{"slug":"llama-65b","modelId":"llama-65b","modelSlug":"llama-65b","name":"LLaMA 65B","organizationSlug":"meta","n":21,"rankEligible":true,"score":98.27,"scoreLo":91.47,"scoreHi":104.1,"modelScoreStateId":"llama-65b::default","modelScoreConfigSlug":"llama-65b","modelScoreStateName":"LLaMA 65B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":732.1911,"rankLo":705.0,"rankHi":761.025},{"slug":"phi-3-5-vision-instruct","modelId":"phi-3-5-vision-instruct","modelSlug":"phi-3-5-vision-instruct","name":"Phi-3.5-vision-instruct","organizationSlug":"microsoft","n":17,"rankEligible":true,"score":98.26,"scoreLo":91.45,"scoreHi":104.13,"modelScoreStateId":"phi-3-5-vision-instruct::default","modelScoreConfigSlug":"phi-3-5-vision-instruct","modelScoreStateName":"Phi-3.5-vision-instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":732.5056,"rankLo":704.0,"rankHi":763.0},{"slug":"lfm2-vl-1-6b","modelId":"lfm2-vl-1-6b","modelSlug":"lfm2-vl-1-6b","name":"LFM2-VL-1.6B","organizationSlug":"liquid-ai","n":11,"rankEligible":true,"score":98.01,"scoreLo":90.59,"scoreHi":104.37,"modelScoreStateId":"lfm2-vl-1-6b::default","modelScoreConfigSlug":"lfm2-vl-1-6b","modelScoreStateName":"LFM2-VL-1.6B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":734.3961,"rankLo":699.0,"rankHi":772.0},{"slug":"vicuna-33b-v1-3","modelId":"vicuna-33b-v1-3","modelSlug":"vicuna-33b-v1-3","name":"Vicuna 33B v1.3","organizationSlug":"lmsys","n":7,"rankEligible":true,"score":97.94,"scoreLo":90.11,"scoreHi":104.31,"modelScoreStateId":"vicuna-33b-v1-3::default","modelScoreConfigSlug":"vicuna-33b-v1-3","modelScoreStateName":"Vicuna 33B v1.3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":734.8508,"rankLo":700.0,"rankHi":770.0},{"slug":"qwen2-5-1-5b-instruct","modelId":"qwen2-5-1-5b-instruct","modelSlug":"qwen2-5-1-5b-instruct","name":"Qwen2.5 1.5B Instruct","organizationSlug":"alibaba","n":11,"rankEligible":true,"score":97.86,"scoreLo":90.18,"scoreHi":104.36,"modelScoreStateId":"qwen2-5-1-5b-instruct::default","modelScoreConfigSlug":"qwen2-5-1-5b-instruct","modelScoreStateName":"Qwen2.5 1.5B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":735.5264,"rankLo":700.0,"rankHi":770.0},{"slug":"deepseek-v2-lite-chat","modelId":"deepseek-v2-lite-chat","modelSlug":"deepseek-v2-lite-chat","name":"DeepSeek-V2-Lite-Chat","organizationSlug":"deepseek","n":16,"rankEligible":true,"score":97.74,"scoreLo":90.6,"scoreHi":103.63,"modelScoreStateId":"deepseek-v2-lite-chat::default","modelScoreConfigSlug":"deepseek-v2-lite-chat","modelScoreStateName":"DeepSeek-V2-Lite-Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":736.4536,"rankLo":711.0,"rankHi":763.0},{"slug":"mistral-7b-v0-1","modelId":"mistral-7b-v0-1","modelSlug":"mistral-7b-v0-1","name":"Mistral 7B v0.1 Base","organizationSlug":"mistral-ai","n":40,"rankEligible":true,"score":97.59,"scoreLo":91.44,"scoreHi":102.75,"modelScoreStateId":"mistral-7b-v0-1::default","modelScoreConfigSlug":"mistral-7b-v0-1","modelScoreStateName":"Mistral 7B v0.1 Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":737.3797,"rankLo":724.0,"rankHi":751.0},{"slug":"lfm2-2-6b","modelId":"lfm2-2-6b","modelSlug":"lfm2-2-6b","name":"LFM2-2.6B","organizationSlug":"liquid-ai","n":8,"rankEligible":true,"score":97.55,"scoreLo":88.1,"scoreHi":105.6,"modelScoreStateId":"lfm2-2-6b::disabled","modelScoreConfigSlug":"lfm2-2-6b-no-thinking","modelScoreStateName":"LFM2-2.6B (disabled)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"disabled","modelScoreReasoningEffort":"none","modelScoreHarness":"none","rankMean":737.4272,"rankLo":684.0,"rankHi":789.0},{"slug":"nemotron-4-15b","modelId":"nemotron-4-15b","modelSlug":"nemotron-4-15b","name":"Nemotron-4 15B","organizationSlug":"nvidia","n":10,"rankEligible":true,"score":97.55,"scoreLo":89.79,"scoreHi":103.89,"modelScoreStateId":"nemotron-4-15b::default","modelScoreConfigSlug":"nemotron-4-15b","modelScoreStateName":"Nemotron-4 15B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":738.0467,"rankLo":704.0,"rankHi":773.0},{"slug":"yi-6b","modelId":"yi-6b","modelSlug":"yi-6b","name":"Yi-6B","organizationSlug":"01-ai","n":11,"rankEligible":true,"score":97.48,"scoreLo":90.31,"scoreHi":103.41,"modelScoreStateId":"yi-6b::default","modelScoreConfigSlug":"yi-6b","modelScoreStateName":"Yi-6B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":738.5531,"rankLo":710.0,"rankHi":768.0},{"slug":"mistral-7b-instruct-v0-2","modelId":"mistral-7b-instruct-v0-2","modelSlug":"mistral-7b-instruct-v0-2","name":"Mistral 7B Instruct v0.2","organizationSlug":"mistral-ai","n":20,"rankEligible":true,"score":97.45,"scoreLo":90.63,"scoreHi":103.02,"modelScoreStateId":"mistral-7b-instruct-v0-2::default","modelScoreConfigSlug":"mistral-7b-instruct-v0-2","modelScoreStateName":"Mistral 7B Instruct v0.2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":738.7592,"rankLo":716.0,"rankHi":763.0},{"slug":"internlm-20b","modelId":"internlm-20b","modelSlug":"internlm-20b","name":"InternLM 20B","organizationSlug":"shanghai-ai-lab","n":17,"rankEligible":true,"score":97.1,"scoreLo":90.07,"scoreHi":103.12,"modelScoreStateId":"internlm-20b::default","modelScoreConfigSlug":"internlm-20b","modelScoreStateName":"InternLM 20B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":741.64,"rankLo":714.0,"rankHi":768.0},{"slug":"qwen3-0-6b","modelId":"qwen3-0-6b","modelSlug":"qwen3-0-6b","name":"Qwen3 0.6B","organizationSlug":"alibaba","n":21,"rankEligible":true,"score":97.08,"scoreLo":90.11,"scoreHi":102.93,"modelScoreStateId":"qwen3-0-6b::enabled","modelScoreConfigSlug":"qwen3-0-6b-thinking","modelScoreStateName":"Qwen3 0.6B (thinking)","modelScoreStateBestProbability":0.9980555555555556,"modelScoreReasoningCondition":"enabled","modelScoreReasoningEffort":"thinking","modelScoreHarness":"none","rankMean":741.8908,"rankLo":715.0,"rankHi":769.0},{"slug":"zephyr-7b-alpha","modelId":"zephyr-7b-alpha","modelSlug":"zephyr-7b-alpha","name":"Zephyr 7B Alpha","organizationSlug":"hugging-face","n":13,"rankEligible":true,"score":96.84,"scoreLo":89.81,"scoreHi":102.9,"modelScoreStateId":"zephyr-7b-alpha::default","modelScoreConfigSlug":"zephyr-7b-alpha","modelScoreStateName":"Zephyr 7B Alpha","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":743.7353,"rankLo":716.0,"rankHi":771.0},{"slug":"zephyr-7b-beta","modelId":"zephyr-7b-beta","modelSlug":"zephyr-7b-beta","name":"Zephyr 7B Beta","organizationSlug":"hugging-face","n":25,"rankEligible":true,"score":96.82,"scoreLo":90.22,"scoreHi":102.15,"modelScoreStateId":"zephyr-7b-beta::default","modelScoreConfigSlug":"zephyr-7b-beta","modelScoreStateName":"Zephyr 7B Beta","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":744.1089,"rankLo":729.0,"rankHi":759.0},{"slug":"mistral-7b-instruct-v0-3","modelId":"mistral-7b-instruct-v0-3","modelSlug":"mistral-7b-instruct-v0-3","name":"Mistral 7B Instruct v0.3","organizationSlug":"mistral-ai","n":28,"rankEligible":true,"score":96.81,"scoreLo":89.95,"scoreHi":102.75,"modelScoreStateId":"mistral-7b-instruct-v0-3::default","modelScoreConfigSlug":"mistral-7b-instruct-v0-3","modelScoreStateName":"Mistral 7B Instruct v0.3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":743.9942,"rankLo":719.0,"rankHi":769.0},{"slug":"deepseek-math-7b-instruct","modelId":"deepseek-math-7b-instruct","modelSlug":"deepseek-math-7b-instruct","name":"DeepSeekMath 7B Instruct","organizationSlug":"deepseek","n":8,"rankEligible":true,"score":96.77,"scoreLo":88.78,"scoreHi":103.73,"modelScoreStateId":"deepseek-math-7b-instruct::default","modelScoreConfigSlug":"deepseek-math-7b-instruct","modelScoreStateName":"DeepSeekMath 7B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":744.0892,"rankLo":705.0,"rankHi":780.0},{"slug":"qwen1-5-7b","modelId":"qwen1-5-7b","modelSlug":"qwen1-5-7b","name":"Qwen1.5 7B","organizationSlug":"alibaba","n":33,"rankEligible":true,"score":96.72,"scoreLo":90.36,"scoreHi":102.04,"modelScoreStateId":"qwen1-5-7b::default","modelScoreConfigSlug":"qwen1-5-7b","modelScoreStateName":"Qwen1.5 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":744.9397,"rankLo":730.0,"rankHi":759.0},{"slug":"gemma-3n-e4b","modelId":"gemma-3n-e4b","modelSlug":"gemma-3n-e4b","name":"Gemma 3n E4B","organizationSlug":"google","n":9,"rankEligible":true,"score":96.44,"scoreLo":88.35,"scoreHi":103.52,"modelScoreStateId":"gemma-3n-e4b::default","modelScoreConfigSlug":"gemma-3n-e4b","modelScoreStateName":"Gemma 3n E4B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":746.5964,"rankLo":705.0,"rankHi":784.0},{"slug":"qwen1-5-7b-chat","modelId":"qwen1-5-7b-chat","modelSlug":"qwen1-5-7b-chat","name":"Qwen1.5 7B Chat","organizationSlug":"alibaba","n":35,"rankEligible":true,"score":96.15,"scoreLo":89.61,"scoreHi":101.52,"modelScoreStateId":"qwen1-5-7b-chat::default","modelScoreConfigSlug":"qwen1-5-7b-chat","modelScoreStateName":"Qwen1.5 7B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":749.9325,"rankLo":736.0,"rankHi":764.0},{"slug":"llama-2-70b-chat-hf","modelId":"llama-2-70b-chat-hf","modelSlug":"llama-2-70b-chat-hf","name":"Llama 2 70B Chat","organizationSlug":"meta","n":26,"rankEligible":true,"score":96.07,"scoreLo":89.03,"scoreHi":101.94,"modelScoreStateId":"llama-2-70b-chat-hf::default","modelScoreConfigSlug":"llama-2-70b-chat-hf","modelScoreStateName":"Llama 2 70B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":750.1319,"rankLo":727.0,"rankHi":771.0},{"slug":"deepseek-math-7b-base","modelId":"deepseek-math-7b-base","modelSlug":"deepseek-math-7b-base","name":"DeepSeekMath 7B Base","organizationSlug":"deepseek","n":12,"rankEligible":true,"score":95.88,"scoreLo":88.49,"scoreHi":102.46,"modelScoreStateId":"deepseek-math-7b-base::default","modelScoreConfigSlug":"deepseek-math-7b-base","modelScoreStateName":"DeepSeekMath 7B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":751.2736,"rankLo":719.0,"rankHi":780.0},{"slug":"yi-6b-chat","modelId":"yi-6b-chat","modelSlug":"yi-6b-chat","name":"Yi 6B Chat","organizationSlug":"01-ai","n":9,"rankEligible":true,"score":95.74,"scoreLo":87.81,"scoreHi":102.58,"modelScoreStateId":"yi-6b-chat::default","modelScoreConfigSlug":"yi-6b-chat","modelScoreStateName":"Yi 6B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":752.1881,"rankLo":719.0,"rankHi":784.0},{"slug":"mythomax-l2-13b","modelId":"mythomax-l2-13b","modelSlug":"mythomax-l2-13b","name":"MythoMax L2 13B","organizationSlug":"gryphe","n":6,"rankEligible":true,"score":95.64,"scoreLo":86.85,"scoreHi":103.26,"modelScoreStateId":"mythomax-l2-13b::default","modelScoreConfigSlug":"mythomax-l2-13b","modelScoreStateName":"MythoMax L2 13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":752.5297,"rankLo":709.0,"rankHi":790.0},{"slug":"gopher-280b","modelId":"gopher-280b","modelSlug":"gopher-280b","name":"Gopher (280B)","organizationSlug":"google","n":8,"rankEligible":true,"score":95.56,"scoreLo":86.71,"scoreHi":103.42,"modelScoreStateId":"gopher-280b::default","modelScoreConfigSlug":"gopher-280b","modelScoreStateName":"Gopher (280B)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":752.8564,"rankLo":705.975,"rankHi":794.0},{"slug":"qwen1-5-moe-a2-7b","modelId":"qwen1-5-moe-a2-7b","modelSlug":"qwen1-5-moe-a2-7b","name":"Qwen1.5 MoE A2.7B","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":95.56,"scoreLo":88.13,"scoreHi":101.79,"modelScoreStateId":"qwen1-5-moe-a2-7b::default","modelScoreConfigSlug":"qwen1-5-moe-a2-7b","modelScoreStateName":"Qwen1.5 MoE A2.7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":753.7561,"rankLo":723.0,"rankHi":781.0},{"slug":"qwen1-5-moe-a2-7b-chat","modelId":"qwen1-5-moe-a2-7b-chat","modelSlug":"qwen1-5-moe-a2-7b-chat","name":"Qwen1.5 MoE A2.7B Chat","organizationSlug":"alibaba","n":14,"rankEligible":true,"score":95.52,"scoreLo":88.11,"scoreHi":101.64,"modelScoreStateId":"qwen1-5-moe-a2-7b-chat::default","modelScoreConfigSlug":"qwen1-5-moe-a2-7b-chat","modelScoreStateName":"Qwen1.5 MoE A2.7B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":754.2133,"rankLo":727.975,"rankHi":777.0},{"slug":"gemma-1-1-7b-it","modelId":"gemma-1-1-7b-it","modelSlug":"gemma-1-1-7b-it","name":"Gemma 1.1 7B IT","organizationSlug":"google","n":25,"rankEligible":true,"score":95.45,"scoreLo":88.81,"scoreHi":100.95,"modelScoreStateId":"gemma-1-1-7b-it::default","modelScoreConfigSlug":"gemma-1-1-7b-it","modelScoreStateName":"Gemma 1.1 7B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":755.6581,"rankLo":741.0,"rankHi":769.0},{"slug":"llama-33b","modelId":"llama-33b","modelSlug":"llama-33b","name":"LLaMA 33B","organizationSlug":"meta","n":16,"rankEligible":true,"score":95.39,"scoreLo":87.86,"scoreHi":101.68,"modelScoreStateId":"llama-33b::default","modelScoreConfigSlug":"llama-33b","modelScoreStateName":"LLaMA 33B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":754.9586,"rankLo":724.0,"rankHi":783.0},{"slug":"qwen2-5-1-5b","modelId":"qwen2-5-1-5b","modelSlug":"qwen2-5-1-5b","name":"Qwen2.5 1.5B","organizationSlug":"alibaba","n":16,"rankEligible":true,"score":95.35,"scoreLo":88.02,"scoreHi":101.57,"modelScoreStateId":"qwen2-5-1-5b::default","modelScoreConfigSlug":"qwen2-5-1-5b","modelScoreStateName":"Qwen2.5 1.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":755.3969,"rankLo":727.0,"rankHi":781.0},{"slug":"recurrentgemma-9b","modelId":"recurrentgemma-9b","modelSlug":"recurrentgemma-9b","name":"RecurrentGemma 9B","organizationSlug":"google","n":18,"rankEligible":true,"score":95.3,"scoreLo":88.0,"scoreHi":101.33,"modelScoreStateId":"recurrentgemma-9b::default","modelScoreConfigSlug":"recurrentgemma-9b","modelScoreStateName":"RecurrentGemma 9B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":755.8914,"rankLo":727.0,"rankHi":780.0},{"slug":"lfm2-5-350m","modelId":"lfm2-5-350m","modelSlug":"lfm2-5-350m","name":"LFM2.5-350M","organizationSlug":"liquid-ai","n":7,"rankEligible":true,"score":95.09,"scoreLo":85.1,"scoreHi":103.62,"modelScoreStateId":"lfm2-5-350m::default","modelScoreConfigSlug":"lfm2-5-350m","modelScoreStateName":"LFM2.5-350M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":755.7056,"rankLo":703.0,"rankHi":803.0},{"slug":"vicuna-13b-v1-5","modelId":"vicuna-13b-v1-5","modelSlug":"vicuna-13b-v1-5","name":"Vicuna 13B v1.5","organizationSlug":"lmsys","n":6,"rankEligible":true,"score":95.06,"scoreLo":86.92,"scoreHi":101.98,"modelScoreStateId":"vicuna-13b-v1-5::default","modelScoreConfigSlug":"vicuna-13b-v1-5","modelScoreStateName":"Vicuna 13B v1.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":757.1036,"rankLo":719.975,"rankHi":790.0},{"slug":"deepseek-math-7b-rl","modelId":"deepseek-math-7b-rl","modelSlug":"deepseek-math-7b-rl","name":"DeepSeekMath 7B RL","organizationSlug":"deepseek","n":9,"rankEligible":true,"score":94.95,"scoreLo":87.5,"scoreHi":101.71,"modelScoreStateId":"deepseek-math-7b-rl::default","modelScoreConfigSlug":"deepseek-math-7b-rl","modelScoreStateName":"DeepSeekMath 7B RL","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":758.1389,"rankLo":725.0,"rankHi":786.0},{"slug":"olmo-2-1124-7b-instruct","modelId":"olmo-2-1124-7b-instruct","modelSlug":"olmo-2-1124-7b-instruct","name":"OLMo 2 1124 7B Instruct","organizationSlug":"allen-ai","n":10,"rankEligible":true,"score":94.73,"scoreLo":86.0,"scoreHi":102.48,"modelScoreStateId":"olmo-2-1124-7b-instruct::default","modelScoreConfigSlug":"olmo-2-1124-7b-instruct","modelScoreStateName":"OLMo 2 1124 7B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":758.8806,"rankLo":715.0,"rankHi":799.0},{"slug":"codellama-34b-hf","modelId":"codellama-34b-hf","modelSlug":"codellama-34b-hf","name":"Code Llama 34B","organizationSlug":"meta","n":11,"rankEligible":true,"score":94.45,"scoreLo":86.08,"scoreHi":101.66,"modelScoreStateId":"codellama-34b-hf::default","modelScoreConfigSlug":"codellama-34b-hf","modelScoreStateName":"Code Llama 34B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":761.2583,"rankLo":722.0,"rankHi":795.0},{"slug":"falcon-11b","modelId":"falcon-11b","modelSlug":"falcon-11b","name":"Falcon2-11B","organizationSlug":"tii","n":14,"rankEligible":true,"score":94.24,"scoreLo":86.23,"scoreHi":100.84,"modelScoreStateId":"falcon-11b::default","modelScoreConfigSlug":"falcon-11b","modelScoreStateName":"Falcon2-11B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":763.3314,"rankLo":731.975,"rankHi":790.0},{"slug":"gemma-2-2b-it","modelId":"gemma-2-2b-it","modelSlug":"gemma-2-2b-it","name":"Gemma 2 2B IT","organizationSlug":"google","n":25,"rankEligible":true,"score":94.2,"scoreLo":87.08,"scoreHi":100.15,"modelScoreStateId":"gemma-2-2b-it::default","modelScoreConfigSlug":"gemma-2-2b-it","modelScoreStateName":"Gemma 2 2B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":764.285,"rankLo":743.0,"rankHi":783.0},{"slug":"codellama-34b-python-hf","modelId":"codellama-34b-python-hf","modelSlug":"codellama-34b-python-hf","name":"Code Llama Python 34B","organizationSlug":"meta","n":12,"rankEligible":true,"score":93.6,"scoreLo":85.45,"scoreHi":100.75,"modelScoreStateId":"codellama-34b-python-hf::default","modelScoreConfigSlug":"codellama-34b-python-hf","modelScoreStateName":"Code Llama Python 34B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":767.265,"rankLo":729.975,"rankHi":799.0},{"slug":"baichuan2-13b-chat","modelId":"baichuan2-13b-chat","modelSlug":"baichuan2-13b-chat","name":"Baichuan2-13B-Chat","organizationSlug":"baichuan","n":5,"rankEligible":true,"score":93.43,"scoreLo":84.44,"scoreHi":101.22,"modelScoreStateId":"baichuan2-13b-chat::default","modelScoreConfigSlug":"baichuan2-13b-chat","modelScoreStateName":"Baichuan2-13B-Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":767.9856,"rankLo":723.975,"rankHi":805.0},{"slug":"phi-2","modelId":"phi-2","modelSlug":"phi-2","name":"Phi-2","organizationSlug":"microsoft","n":27,"rankEligible":true,"score":93.29,"scoreLo":85.79,"scoreHi":99.45,"modelScoreStateId":"phi-2::default","modelScoreConfigSlug":"phi-2","modelScoreStateName":"Phi-2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":770.2883,"rankLo":750.0,"rankHi":788.0},{"slug":"qwen-7b-chat","modelId":"qwen-7b-chat","modelSlug":"qwen-7b-chat","name":"Qwen 7B Chat","organizationSlug":"alibaba","n":5,"rankEligible":true,"score":93.29,"scoreLo":84.03,"scoreHi":101.05,"modelScoreStateId":"qwen-7b-chat::default","modelScoreConfigSlug":"qwen-7b-chat","modelScoreStateName":"Qwen 7B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":768.8822,"rankLo":725.0,"rankHi":806.0},{"slug":"codellama-34b-instruct-hf","modelId":"codellama-34b-instruct-hf","modelSlug":"codellama-34b-instruct-hf","name":"Code Llama Instruct 34B","organizationSlug":"meta","n":14,"rankEligible":true,"score":92.87,"scoreLo":84.6,"scoreHi":99.72,"modelScoreStateId":"codellama-34b-instruct-hf::default","modelScoreConfigSlug":"codellama-34b-instruct-hf","modelScoreStateName":"Code Llama Instruct 34B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":772.3292,"rankLo":740.0,"rankHi":799.0},{"slug":"llama-3-2-3b","modelId":"llama-3-2-3b","modelSlug":"llama-3-2-3b","name":"Llama 3.2 3B","organizationSlug":"meta","n":8,"rankEligible":true,"score":92.81,"scoreLo":83.93,"scoreHi":100.26,"modelScoreStateId":"llama-3-2-3b::default","modelScoreConfigSlug":"llama-3-2-3b","modelScoreStateName":"Llama 3.2 3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":772.3403,"rankLo":735.0,"rankHi":805.0},{"slug":"lfm2-1-2b","modelId":"lfm2-1-2b","modelSlug":"lfm2-1-2b","name":"LFM2-1.2B","organizationSlug":"liquid-ai","n":9,"rankEligible":true,"score":92.79,"scoreLo":83.6,"scoreHi":100.66,"modelScoreStateId":"lfm2-1-2b::default","modelScoreConfigSlug":"lfm2-1-2b","modelScoreStateName":"LFM2-1.2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":772.2739,"rankLo":729.0,"rankHi":807.0},{"slug":"qwen-7b","modelId":"qwen-7b","modelSlug":"qwen-7b","name":"Qwen 7B","organizationSlug":"alibaba","n":17,"rankEligible":true,"score":92.74,"scoreLo":84.97,"scoreHi":99.08,"modelScoreStateId":"qwen-7b::default","modelScoreConfigSlug":"qwen-7b","modelScoreStateName":"Qwen 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":773.5608,"rankLo":749.0,"rankHi":797.0},{"slug":"lfm2-5-230m","modelId":"lfm2-5-230m","modelSlug":"lfm2-5-230m","name":"LFM2.5-230M","organizationSlug":"liquid-ai","n":7,"rankEligible":true,"score":92.73,"scoreLo":82.23,"scoreHi":101.37,"modelScoreStateId":"lfm2-5-230m::default","modelScoreConfigSlug":"lfm2-5-230m","modelScoreStateName":"LFM2.5-230M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":771.7181,"rankLo":720.0,"rankHi":813.0},{"slug":"gemma-3-4b-pt","modelId":"gemma-3-4b-pt","modelSlug":"gemma-3-4b-pt","name":"Gemma 3 4B PT","organizationSlug":"google","n":21,"rankEligible":true,"score":92.7,"scoreLo":84.87,"scoreHi":98.93,"modelScoreStateId":"gemma-3-4b-pt::default","modelScoreConfigSlug":"gemma-3-4b-pt","modelScoreStateName":"Gemma 3 4B PT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":773.9069,"rankLo":752.0,"rankHi":795.0},{"slug":"mistral-7b-instruct-v0-1","modelId":"mistral-7b-instruct-v0-1","modelSlug":"mistral-7b-instruct-v0-1","name":"Mistral 7B Instruct v0.1","organizationSlug":"mistral-ai","n":32,"rankEligible":true,"score":92.64,"scoreLo":85.72,"scoreHi":98.43,"modelScoreStateId":"mistral-7b-instruct-v0-1::default","modelScoreConfigSlug":"mistral-7b-instruct-v0-1","modelScoreStateName":"Mistral 7B Instruct v0.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":774.5267,"rankLo":763.0,"rankHi":787.0},{"slug":"codellama-13b-instruct-hf","modelId":"codellama-13b-instruct-hf","modelSlug":"codellama-13b-instruct-hf","name":"Code Llama Instruct 13B","organizationSlug":"meta","n":10,"rankEligible":true,"score":92.15,"scoreLo":83.36,"scoreHi":99.53,"modelScoreStateId":"codellama-13b-instruct-hf::default","modelScoreConfigSlug":"codellama-13b-instruct-hf","modelScoreStateName":"Code Llama Instruct 13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":776.5372,"rankLo":739.0,"rankHi":807.0},{"slug":"deepseek-vl-7b-chat","modelId":"deepseek-vl-7b-chat","modelSlug":"deepseek-vl-7b-chat","name":"DeepSeek-VL 7B Chat","organizationSlug":"deepseek","n":12,"rankEligible":true,"score":92.06,"scoreLo":83.95,"scoreHi":99.0,"modelScoreStateId":"deepseek-vl-7b-chat::default","modelScoreConfigSlug":"deepseek-vl-7b","modelScoreStateName":"DeepSeek-VL 7B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":777.5625,"rankLo":750.975,"rankHi":802.0},{"slug":"qwen2-5-coder-1-5b","modelId":"qwen2-5-coder-1-5b","modelSlug":"qwen2-5-coder-1-5b","name":"Qwen2.5 Coder 1.5B","organizationSlug":"alibaba","n":14,"rankEligible":true,"score":91.59,"scoreLo":83.52,"scoreHi":98.41,"modelScoreStateId":"qwen2-5-coder-1-5b::default","modelScoreConfigSlug":"qwen2-5-coder-1-5b","modelScoreStateName":"Qwen2.5 Coder 1.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":780.5756,"rankLo":753.0,"rankHi":805.0},{"slug":"deepseek-v2-lite","modelId":"deepseek-v2-lite","modelSlug":"deepseek-v2-lite","name":"DeepSeek-V2-Lite","organizationSlug":"deepseek","n":13,"rankEligible":true,"score":91.02,"scoreLo":82.52,"scoreHi":98.36,"modelScoreStateId":"deepseek-v2-lite::default","modelScoreConfigSlug":"deepseek-v2-lite","modelScoreStateName":"DeepSeek-V2-Lite","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":783.8689,"rankLo":757.0,"rankHi":808.0},{"slug":"falcon-40b-instruct","modelId":"falcon-40b-instruct","modelSlug":"falcon-40b-instruct","name":"Falcon 40B Instruct","organizationSlug":"tii","n":10,"rankEligible":true,"score":91.02,"scoreLo":82.43,"scoreHi":98.07,"modelScoreStateId":"falcon-40b-instruct::default","modelScoreConfigSlug":"falcon-40b-instruct","modelScoreStateName":"Falcon 40B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":783.7489,"rankLo":756.0,"rankHi":808.025},{"slug":"qwen-vl-chat","modelId":"qwen-vl-chat","modelSlug":"qwen-vl-chat","name":"Qwen-VL-Chat","organizationSlug":"alibaba","n":4,"rankEligible":true,"score":91.0,"scoreLo":79.84,"scoreHi":100.47,"modelScoreStateId":"qwen-vl-chat::default","modelScoreConfigSlug":"qwen-vl-chat","modelScoreStateName":"Qwen-VL-Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":782.0369,"rankLo":727.0,"rankHi":825.0},{"slug":"starcoder2-15b","modelId":"starcoder2-15b","modelSlug":"starcoder2-15b","name":"StarCoder2-15B","organizationSlug":"hugging-face","n":25,"rankEligible":true,"score":90.69,"scoreLo":82.92,"scoreHi":97.28,"modelScoreStateId":"starcoder2-15b::default","modelScoreConfigSlug":"starcoder2-15b","modelScoreStateName":"StarCoder2-15B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":786.1311,"rankLo":767.0,"rankHi":804.0},{"slug":"qwen3-0-6b-base","modelId":"qwen3-0-6b-base","modelSlug":"qwen3-0-6b-base","name":"Qwen3 0.6B Base","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":90.62,"scoreLo":82.46,"scoreHi":97.44,"modelScoreStateId":"qwen3-0-6b-base::default","modelScoreConfigSlug":"qwen3-0-6b-base","modelScoreStateName":"Qwen3 0.6B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":786.3358,"rankLo":764.0,"rankHi":808.0},{"slug":"deepseek-coder-7b-base-v1-5","modelId":"deepseek-coder-7b-base-v1-5","modelSlug":"deepseek-coder-7b-base-v1-5","name":"DeepSeek-Coder-Base 7B v1.5","organizationSlug":"deepseek","n":8,"rankEligible":true,"score":90.18,"scoreLo":80.45,"scoreHi":98.46,"modelScoreStateId":"deepseek-coder-7b-base-v1-5::default","modelScoreConfigSlug":"deepseek-coder-7b-base-v1-5","modelScoreStateName":"DeepSeek-Coder-Base 7B v1.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":787.9503,"rankLo":750.0,"rankHi":819.0},{"slug":"falcon-40b","modelId":"falcon-40b","modelSlug":"falcon-40b","name":"Falcon 40B","organizationSlug":"tii","n":29,"rankEligible":true,"score":90.13,"scoreLo":82.57,"scoreHi":96.48,"modelScoreStateId":"falcon-40b::default","modelScoreConfigSlug":"falcon-40b","modelScoreStateName":"Falcon 40B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":789.38,"rankLo":772.0,"rankHi":806.0},{"slug":"deepseek-llm-7b-chat","modelId":"deepseek-llm-7b-chat","modelSlug":"deepseek-llm-7b-chat","name":"DeepSeek LLM 7B Chat","organizationSlug":"deepseek","n":22,"rankEligible":true,"score":90.0,"scoreLo":81.98,"scoreHi":96.71,"modelScoreStateId":"deepseek-llm-7b-chat::default","modelScoreConfigSlug":"deepseek-llm-7b-chat","modelScoreStateName":"DeepSeek LLM 7B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":790.0019,"rankLo":770.0,"rankHi":808.0},{"slug":"deepseek-moe-16b-chat","modelId":"deepseek-moe-16b-chat","modelSlug":"deepseek-moe-16b-chat","name":"DeepSeekMoE 16B Chat","organizationSlug":"deepseek","n":18,"rankEligible":true,"score":89.99,"scoreLo":81.69,"scoreHi":96.96,"modelScoreStateId":"deepseek-moe-16b-chat::default","modelScoreConfigSlug":"deepseek-moe-16b-chat","modelScoreStateName":"DeepSeekMoE 16B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":789.8017,"rankLo":765.0,"rankHi":811.025},{"slug":"gemma-7b-it","modelId":"gemma-7b-it","modelSlug":"gemma-7b-it","name":"Gemma 7B IT","organizationSlug":"google","n":28,"rankEligible":true,"score":89.96,"scoreLo":82.73,"scoreHi":96.04,"modelScoreStateId":"gemma-7b-it::default","modelScoreConfigSlug":"gemma-7b-it","modelScoreStateName":"Gemma 7B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":790.5292,"rankLo":779.0,"rankHi":802.0},{"slug":"deepseek-coder-7b-instruct-v1-5","modelId":"deepseek-coder-7b-instruct-v1-5","modelSlug":"deepseek-coder-7b-instruct-v1-5","name":"DeepSeek-Coder-Instruct 7B v1.5","organizationSlug":"deepseek","n":14,"rankEligible":true,"score":89.76,"scoreLo":81.04,"scoreHi":97.04,"modelScoreStateId":"deepseek-coder-7b-instruct-v1-5::default","modelScoreConfigSlug":"deepseek-coder-7b-instruct-v1-5","modelScoreStateName":"DeepSeek-Coder-Instruct 7B v1.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":791.0214,"rankLo":767.0,"rankHi":813.0},{"slug":"lfm2-5-vl-450m","modelId":"lfm2-5-vl-450m","modelSlug":"lfm2-5-vl-450m","name":"LFM2.5-VL-450M","organizationSlug":"liquid-ai","n":12,"rankEligible":true,"score":89.58,"scoreLo":80.5,"scoreHi":97.37,"modelScoreStateId":"lfm2-5-vl-450m::default","modelScoreConfigSlug":"lfm2-5-vl-450m","modelScoreStateName":"LFM2.5-VL-450M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":791.76,"rankLo":763.0,"rankHi":818.0},{"slug":"qwen1-5-4b","modelId":"qwen1-5-4b","modelSlug":"qwen1-5-4b","name":"Qwen1.5 4B","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":89.49,"scoreLo":80.83,"scoreHi":96.52,"modelScoreStateId":"qwen1-5-4b::default","modelScoreConfigSlug":"qwen1-5-4b","modelScoreStateName":"Qwen1.5 4B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":792.6903,"rankLo":768.975,"rankHi":813.025},{"slug":"palm-62b","modelId":"palm-62b","modelSlug":"palm-62b","name":"PaLM 62B","organizationSlug":"google","n":15,"rankEligible":true,"score":89.44,"scoreLo":81.04,"scoreHi":96.84,"modelScoreStateId":"palm-62b::default","modelScoreConfigSlug":"palm-62b","modelScoreStateName":"PaLM 62B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":792.7514,"rankLo":767.0,"rankHi":815.0},{"slug":"qwen2-1-5b-instruct","modelId":"qwen2-1-5b-instruct","modelSlug":"qwen2-1-5b-instruct","name":"Qwen2 1.5B Instruct","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":89.36,"scoreLo":80.79,"scoreHi":96.71,"modelScoreStateId":"qwen2-1-5b-instruct::default","modelScoreConfigSlug":"qwen2-1-5b-instruct","modelScoreStateName":"Qwen2 1.5B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":793.2806,"rankLo":771.0,"rankHi":814.0},{"slug":"llama-2-13b-chat-hf","modelId":"llama-2-13b-chat-hf","modelSlug":"llama-2-13b-chat-hf","name":"Llama 2 13B Chat","organizationSlug":"meta","n":24,"rankEligible":true,"score":89.22,"scoreLo":81.06,"scoreHi":95.78,"modelScoreStateId":"llama-2-13b-chat-hf::default","modelScoreConfigSlug":"llama-2-13b-chat-hf","modelScoreStateName":"Llama 2 13B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":794.47,"rankLo":776.0,"rankHi":810.0},{"slug":"vicuna-13b-v1-3","modelId":"vicuna-13b-v1-3","modelSlug":"vicuna-13b-v1-3","name":"Vicuna 13B v1.3","organizationSlug":"lmsys","n":11,"rankEligible":true,"score":89.05,"scoreLo":80.3,"scoreHi":96.32,"modelScoreStateId":"vicuna-13b-v1-3::default","modelScoreConfigSlug":"vicuna-13b-v1-3","modelScoreStateName":"Vicuna 13B v1.3","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":794.8297,"rankLo":769.0,"rankHi":817.0},{"slug":"qwen2-1-5b","modelId":"qwen2-1-5b","modelSlug":"qwen2-1-5b","name":"Qwen2-1.5B","organizationSlug":"alibaba","n":18,"rankEligible":true,"score":88.82,"scoreLo":80.59,"scoreHi":95.79,"modelScoreStateId":"qwen2-1-5b::default","modelScoreConfigSlug":"qwen2-1-5b","modelScoreStateName":"Qwen2-1.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":796.4411,"rankLo":775.0,"rankHi":815.0},{"slug":"baichuan2-13b-base","modelId":"baichuan2-13b-base","modelSlug":"baichuan2-13b-base","name":"Baichuan2-13B-Base","organizationSlug":"baichuan","n":13,"rankEligible":true,"score":88.46,"scoreLo":79.32,"scoreHi":96.09,"modelScoreStateId":"baichuan2-13b-base::default","modelScoreConfigSlug":"baichuan2-13b-base","modelScoreStateName":"Baichuan2-13B-Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":797.8225,"rankLo":771.0,"rankHi":820.025},{"slug":"vicuna-13b-v1-1","modelId":"vicuna-13b-v1-1","modelSlug":"vicuna-13b-v1-1","name":"Vicuna 13B v1.1","organizationSlug":"lmsys","n":10,"rankEligible":true,"score":88.38,"scoreLo":79.44,"scoreHi":95.84,"modelScoreStateId":"vicuna-13b-v1-1::default","modelScoreConfigSlug":"vicuna-13b-v1-1","modelScoreStateName":"Vicuna 13B v1.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":798.2781,"rankLo":772.0,"rankHi":821.025},{"slug":"llama-2-13b-hf","modelId":"llama-2-13b-hf","modelSlug":"llama-2-13b-hf","name":"Llama 2 13B","organizationSlug":"meta","n":37,"rankEligible":true,"score":88.29,"scoreLo":80.26,"scoreHi":95.02,"modelScoreStateId":"llama-2-13b-hf::default","modelScoreConfigSlug":"llama-2-13b-hf","modelScoreStateName":"Llama 2 13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":799.3797,"rankLo":784.0,"rankHi":814.0},{"slug":"gpt-3-175b","modelId":"gpt-3-175b","modelSlug":"gpt-3-175b","name":"GPT-3 175B","organizationSlug":"openai","n":9,"rankEligible":true,"score":88.06,"scoreLo":76.64,"scoreHi":98.1,"modelScoreStateId":"gpt-3-175b::default","modelScoreConfigSlug":"gpt-3-175b","modelScoreStateName":"GPT-3 175B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":798.1139,"rankLo":745.0,"rankHi":834.0},{"slug":"codellama-13b-hf","modelId":"codellama-13b-hf","modelSlug":"codellama-13b-hf","name":"Code Llama 13B","organizationSlug":"meta","n":11,"rankEligible":true,"score":87.75,"scoreLo":78.37,"scoreHi":95.7,"modelScoreStateId":"codellama-13b-hf::default","modelScoreConfigSlug":"codellama-13b-hf","modelScoreStateName":"Code Llama 13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":801.09,"rankLo":772.0,"rankHi":826.0},{"slug":"codellama-13b-python-hf","modelId":"codellama-13b-python-hf","modelSlug":"codellama-13b-python-hf","name":"Code Llama Python 13B","organizationSlug":"meta","n":12,"rankEligible":true,"score":87.71,"scoreLo":78.19,"scoreHi":95.81,"modelScoreStateId":"codellama-13b-python-hf::default","modelScoreConfigSlug":"codellama-13b-python-hf","modelScoreStateName":"Code Llama Python 13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":801.2153,"rankLo":772.0,"rankHi":826.0},{"slug":"gemma-3n-e2b","modelId":"gemma-3n-e2b","modelSlug":"gemma-3n-e2b","name":"Gemma 3n E2B","organizationSlug":"google","n":9,"rankEligible":true,"score":87.24,"scoreLo":77.16,"scoreHi":95.61,"modelScoreStateId":"gemma-3n-e2b::default","modelScoreConfigSlug":"gemma-3n-e2b","modelScoreStateName":"Gemma 3n E2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":803.4406,"rankLo":771.0,"rankHi":830.0},{"slug":"codeqwen1-5-7b-chat","modelId":"codeqwen1-5-7b-chat","modelSlug":"codeqwen1-5-7b-chat","name":"CodeQwen1.5 7B Chat","organizationSlug":"alibaba","n":12,"rankEligible":true,"score":86.79,"scoreLo":77.01,"scoreHi":95.34,"modelScoreStateId":"codeqwen1-5-7b-chat::default","modelScoreConfigSlug":"codeqwen1-5-7b-chat","modelScoreStateName":"CodeQwen1.5 7B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":805.5894,"rankLo":775.0,"rankHi":831.0},{"slug":"deepseek-r1-distill-qwen-1-5b","modelId":"deepseek-r1-distill-qwen-1-5b","modelSlug":"deepseek-r1-distill-qwen-1-5b","name":"DeepSeek-R1-Distill-Qwen-1.5B","organizationSlug":"deepseek","n":26,"rankEligible":true,"score":86.79,"scoreLo":77.56,"scoreHi":94.5,"modelScoreStateId":"deepseek-r1-distill-qwen-1-5b::default","modelScoreConfigSlug":"deepseek-r1-distill-qwen-1-5b","modelScoreStateName":"DeepSeek-R1-Distill-Qwen-1.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":805.9383,"rankLo":781.0,"rankHi":827.0},{"slug":"codellama-7b-instruct-hf","modelId":"codellama-7b-instruct-hf","modelSlug":"codellama-7b-instruct-hf","name":"Code Llama Instruct 7B","organizationSlug":"meta","n":10,"rankEligible":true,"score":86.3,"scoreLo":76.13,"scoreHi":94.56,"modelScoreStateId":"codellama-7b-instruct-hf::default","modelScoreConfigSlug":"codellama-7b-instruct-hf","modelScoreStateName":"Code Llama Instruct 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":807.8569,"rankLo":779.0,"rankHi":833.0},{"slug":"mpt-30b","modelId":"mpt-30b","modelSlug":"mpt-30b","name":"MPT-30B","organizationSlug":"mosaicml","n":21,"rankEligible":true,"score":86.18,"scoreLo":77.74,"scoreHi":93.27,"modelScoreStateId":"mpt-30b::default","modelScoreConfigSlug":"mpt-30b","modelScoreStateName":"MPT-30B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":809.0614,"rankLo":793.0,"rankHi":825.0},{"slug":"granite-4-0-h-350m","modelId":"granite-4-0-h-350m","modelSlug":"granite-4-0-h-350m","name":"Granite 4.0 H 350M","organizationSlug":"ibm","n":12,"rankEligible":true,"score":85.29,"scoreLo":74.03,"scoreHi":94.53,"modelScoreStateId":"granite-4-0-h-350m::default","modelScoreConfigSlug":"granite-4-0-h-350m","modelScoreStateName":"Granite 4.0 H 350M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":811.8808,"rankLo":776.0,"rankHi":841.0},{"slug":"flan-t5-xxl","modelId":"flan-t5-xxl","modelSlug":"flan-t5-xxl","name":"Flan-T5 XXL","organizationSlug":"google","n":9,"rankEligible":true,"score":84.95,"scoreLo":75.19,"scoreHi":93.19,"modelScoreStateId":"flan-t5-xxl::default","modelScoreConfigSlug":"flan-t5-xxl","modelScoreStateName":"Flan-T5 XXL","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":814.0433,"rankLo":790.0,"rankHi":836.0},{"slug":"gemini-nano-2","modelId":"gemini-nano-2","modelSlug":"gemini-nano-2","name":"Gemini Nano 2","organizationSlug":"google","n":13,"rankEligible":true,"score":84.95,"scoreLo":75.2,"scoreHi":93.07,"modelScoreStateId":"gemini-nano-2::default","modelScoreConfigSlug":"gemini-nano-2","modelScoreStateName":"Gemini Nano 2","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":814.0189,"rankLo":791.0,"rankHi":835.0},{"slug":"deepseek-moe-16b-base","modelId":"deepseek-moe-16b-base","modelSlug":"deepseek-moe-16b-base","name":"DeepSeekMoE 16B Base","organizationSlug":"deepseek","n":24,"rankEligible":true,"score":84.71,"scoreLo":76.1,"scoreHi":92.17,"modelScoreStateId":"deepseek-moe-16b-base::default","modelScoreConfigSlug":"deepseek-moe-16b-base","modelScoreStateName":"DeepSeekMoE 16B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":815.285,"rankLo":799.0,"rankHi":831.0},{"slug":"granite-4-0-350m","modelId":"granite-4-0-350m","modelSlug":"granite-4-0-350m","name":"Granite 4.0 350M","organizationSlug":"ibm","n":12,"rankEligible":true,"score":84.51,"scoreLo":73.26,"scoreHi":94.18,"modelScoreStateId":"granite-4-0-350m::default","modelScoreConfigSlug":"granite-4-0-350m","modelScoreStateName":"Granite 4.0 350M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":815.1472,"rankLo":781.0,"rankHi":845.0},{"slug":"llama-3-2-1b-instruct","modelId":"llama-3-2-1b-instruct","modelSlug":"llama-3-2-1b-instruct","name":"Llama 3.2 1B Instruct","organizationSlug":"meta","n":42,"rankEligible":true,"score":84.41,"scoreLo":75.65,"scoreHi":91.61,"modelScoreStateId":"llama-3-2-1b-instruct::default","modelScoreConfigSlug":"llama-3-2-1b-instruct","modelScoreStateName":"Llama 3.2 1B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":816.5803,"rankLo":802.0,"rankHi":831.0},{"slug":"gemma-3-1b-it","modelId":"gemma-3-1b-it","modelSlug":"gemma-3-1b-it","name":"Gemma 3 1B IT","organizationSlug":"google","n":34,"rankEligible":true,"score":84.07,"scoreLo":74.9,"scoreHi":91.69,"modelScoreStateId":"gemma-3-1b-it::default","modelScoreConfigSlug":"gemma-3-1b-it","modelScoreStateName":"Gemma 3 1B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":817.9603,"rankLo":802.0,"rankHi":834.0},{"slug":"deepseek-llm-7b-base","modelId":"deepseek-llm-7b-base","modelSlug":"deepseek-llm-7b-base","name":"DeepSeek LLM 7B Base","organizationSlug":"deepseek","n":21,"rankEligible":true,"score":83.92,"scoreLo":74.78,"scoreHi":91.62,"modelScoreStateId":"deepseek-llm-7b-base::default","modelScoreConfigSlug":"deepseek-llm-7b-base","modelScoreStateName":"DeepSeek LLM 7B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":818.4383,"rankLo":800.0,"rankHi":836.0},{"slug":"qwen1-5-4b-chat","modelId":"qwen1-5-4b-chat","modelSlug":"qwen1-5-4b-chat","name":"Qwen1.5 4B Chat","organizationSlug":"alibaba","n":15,"rankEligible":true,"score":83.86,"scoreLo":74.44,"scoreHi":91.74,"modelScoreStateId":"qwen1-5-4b-chat::default","modelScoreConfigSlug":"qwen1-5-4b-chat","modelScoreStateName":"Qwen1.5 4B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":818.6289,"rankLo":800.0,"rankHi":837.0},{"slug":"codeqwen1-5-7b","modelId":"codeqwen1-5-7b","modelSlug":"codeqwen1-5-7b","name":"CodeQwen1.5 7B","organizationSlug":"alibaba","n":15,"rankEligible":true,"score":83.71,"scoreLo":74.72,"scoreHi":91.48,"modelScoreStateId":"codeqwen1-5-7b::default","modelScoreConfigSlug":"codeqwen1-5-7b","modelScoreStateName":"CodeQwen1.5 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":819.2711,"rankLo":801.0,"rankHi":837.0},{"slug":"codellama-7b-python-hf","modelId":"codellama-7b-python-hf","modelSlug":"codellama-7b-python-hf","name":"Code Llama Python 7B","organizationSlug":"meta","n":12,"rankEligible":true,"score":83.31,"scoreLo":73.2,"scoreHi":91.97,"modelScoreStateId":"codellama-7b-python-hf::default","modelScoreConfigSlug":"codellama-7b-python-hf","modelScoreStateName":"Code Llama Python 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":820.7058,"rankLo":798.0,"rankHi":843.025},{"slug":"gemma-2-2b","modelId":"gemma-2-2b","modelSlug":"gemma-2-2b","name":"Gemma 2 2B","organizationSlug":"google","n":17,"rankEligible":true,"score":82.92,"scoreLo":73.7,"scoreHi":90.53,"modelScoreStateId":"gemma-2-2b::default","modelScoreConfigSlug":"gemma-2-2b","modelScoreStateName":"Gemma 2 2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":822.5347,"rankLo":807.0,"rankHi":839.0},{"slug":"lfm2-vl-450m","modelId":"lfm2-vl-450m","modelSlug":"lfm2-vl-450m","name":"LFM2-VL-450M","organizationSlug":"liquid-ai","n":15,"rankEligible":true,"score":82.46,"scoreLo":72.3,"scoreHi":90.61,"modelScoreStateId":"lfm2-vl-450m::default","modelScoreConfigSlug":"lfm2-vl-450m","modelScoreStateName":"LFM2-VL-450M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":824.1442,"rankLo":804.0,"rankHi":844.0},{"slug":"codellama-7b-hf","modelId":"codellama-7b-hf","modelSlug":"codellama-7b-hf","name":"Code Llama 7B","organizationSlug":"meta","n":11,"rankEligible":true,"score":82.22,"scoreLo":71.46,"scoreHi":91.38,"modelScoreStateId":"codellama-7b-hf::default","modelScoreConfigSlug":"codellama-7b-hf","modelScoreStateName":"Code Llama 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":824.9206,"rankLo":802.0,"rankHi":846.0},{"slug":"deepseek-coder-33b-base","modelId":"deepseek-coder-33b-base","modelSlug":"deepseek-coder-33b-base","name":"DeepSeek-Coder-Base 33B","organizationSlug":"deepseek","n":14,"rankEligible":true,"score":82.1,"scoreLo":72.46,"scoreHi":90.4,"modelScoreStateId":"deepseek-coder-33b-base::default","modelScoreConfigSlug":"deepseek-coder-33b-base","modelScoreStateName":"DeepSeek-Coder-Base 33B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":825.5233,"rankLo":807.0,"rankHi":843.0},{"slug":"llama-2-7b-chat-hf","modelId":"llama-2-7b-chat-hf","modelSlug":"llama-2-7b-chat-hf","name":"Llama 2 7B Chat","organizationSlug":"meta","n":24,"rankEligible":true,"score":81.83,"scoreLo":72.77,"scoreHi":89.42,"modelScoreStateId":"llama-2-7b-chat-hf::default","modelScoreConfigSlug":"llama-2-7b-chat-hf","modelScoreStateName":"Llama 2 7B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":826.7597,"rankLo":814.0,"rankHi":839.0},{"slug":"internlm-7b","modelId":"internlm-7b","modelSlug":"internlm-7b","name":"InternLM 7B","organizationSlug":"shanghai-ai-lab","n":13,"rankEligible":true,"score":81.63,"scoreLo":71.7,"scoreHi":90.39,"modelScoreStateId":"internlm-7b::default","modelScoreConfigSlug":"internlm-7b","modelScoreStateName":"InternLM 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":827.2333,"rankLo":806.0,"rankHi":847.0},{"slug":"lfm2-350m","modelId":"lfm2-350m","modelSlug":"lfm2-350m","name":"LFM2-350M","organizationSlug":"liquid-ai","n":9,"rankEligible":true,"score":81.57,"scoreLo":70.68,"scoreHi":90.54,"modelScoreStateId":"lfm2-350m::default","modelScoreConfigSlug":"lfm2-350m","modelScoreStateName":"LFM2-350M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":827.1678,"rankLo":802.0,"rankHi":851.0},{"slug":"llava-v1-5-7b","modelId":"llava-v1-5-7b","modelSlug":"llava-v1-5-7b","name":"LLaVA v1.5 7B","organizationSlug":"llava-authors","n":5,"rankEligible":true,"score":81.51,"scoreLo":68.48,"scoreHi":92.09,"modelScoreStateId":"llava-v1-5-7b::default","modelScoreConfigSlug":"llava-v1-5-7b","modelScoreStateName":"LLaVA v1.5 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":826.6242,"rankLo":790.0,"rankHi":857.0},{"slug":"olmoe-1b-7b-0125-instruct","modelId":"olmoe-1b-7b-0125-instruct","modelSlug":"olmoe-1b-7b-0125-instruct","name":"OLMoE 1B-7B 0125 Instruct","organizationSlug":"allen-ai","n":5,"rankEligible":true,"score":81.45,"scoreLo":70.05,"scoreHi":91.21,"modelScoreStateId":"olmoe-1b-7b-0125-instruct::default","modelScoreConfigSlug":"olmoe-1b-7b-0125-instruct-helm-capabilities","modelScoreStateName":"OLMoE 1B-7B 0125 Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":827.4097,"rankLo":799.0,"rankHi":852.0},{"slug":"deepseek-coder-6-7b-instruct","modelId":"deepseek-coder-6-7b-instruct","modelSlug":"deepseek-coder-6-7b-instruct","name":"DeepSeek-Coder-Instruct 6.7B","organizationSlug":"deepseek","n":17,"rankEligible":true,"score":81.16,"scoreLo":71.06,"scoreHi":89.52,"modelScoreStateId":"deepseek-coder-6-7b-instruct::default","modelScoreConfigSlug":"deepseek-coder-6-7b-instruct","modelScoreStateName":"DeepSeek-Coder-Instruct 6.7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":829.0072,"rankLo":810.0,"rankHi":847.0},{"slug":"qwen-1-8b","modelId":"qwen-1-8b","modelSlug":"qwen-1-8b","name":"Qwen 1.8B","organizationSlug":"alibaba","n":4,"rankEligible":true,"score":81.04,"scoreLo":68.98,"scoreHi":91.35,"modelScoreStateId":"qwen-1-8b::default","modelScoreConfigSlug":"qwen-1-8b","modelScoreStateName":"Qwen 1.8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":828.6122,"rankLo":797.0,"rankHi":856.0},{"slug":"phi-1-5","modelId":"phi-1-5","modelSlug":"phi-1-5","name":"Phi-1.5","organizationSlug":"microsoft","n":28,"rankEligible":true,"score":80.79,"scoreLo":71.6,"scoreHi":88.3,"modelScoreStateId":"phi-1-5::default","modelScoreConfigSlug":"phi-1-5","modelScoreStateName":"Phi-1.5","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":830.7278,"rankLo":820.0,"rankHi":841.0},{"slug":"llama-13b","modelId":"llama-13b","modelSlug":"llama-13b","name":"LLaMA-13B","organizationSlug":"meta","n":21,"rankEligible":true,"score":80.75,"scoreLo":71.05,"scoreHi":88.88,"modelScoreStateId":"llama-13b::default","modelScoreConfigSlug":"llama-13b","modelScoreStateName":"LLaMA-13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":830.6286,"rankLo":814.0,"rankHi":846.0},{"slug":"baichuan2-7b-base","modelId":"baichuan2-7b-base","modelSlug":"baichuan2-7b-base","name":"Baichuan2-7B-Base","organizationSlug":"baichuan","n":13,"rankEligible":true,"score":80.72,"scoreLo":70.62,"scoreHi":89.24,"modelScoreStateId":"baichuan2-7b-base::default","modelScoreConfigSlug":"baichuan2-7b-base","modelScoreStateName":"Baichuan2-7B-Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":830.5428,"rankLo":810.0,"rankHi":849.025},{"slug":"gpt-3-13b","modelId":"gpt-3-13b","modelSlug":"gpt-3-13b","name":"GPT-3 13B","organizationSlug":"openai","n":9,"rankEligible":true,"score":80.48,"scoreLo":68.14,"scoreHi":90.82,"modelScoreStateId":"gpt-3-13b::default","modelScoreConfigSlug":"gpt-3-13b","modelScoreStateName":"GPT-3 13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":830.6028,"rankLo":799.0,"rankHi":856.0},{"slug":"llama-2-7b-hf","modelId":"llama-2-7b-hf","modelSlug":"llama-2-7b-hf","name":"Llama 2 7B","organizationSlug":"meta","n":37,"rankEligible":true,"score":79.62,"scoreLo":70.27,"scoreHi":87.47,"modelScoreStateId":"llama-2-7b-hf::default","modelScoreConfigSlug":"llama-2-7b-hf","modelScoreStateName":"Llama 2 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":834.8322,"rankLo":823.0,"rankHi":845.025},{"slug":"qwen1-5-1-8b-chat","modelId":"qwen1-5-1-8b-chat","modelSlug":"qwen1-5-1-8b-chat","name":"Qwen1.5 1.8B Chat","organizationSlug":"alibaba","n":18,"rankEligible":true,"score":78.17,"scoreLo":67.87,"scoreHi":86.73,"modelScoreStateId":"qwen1-5-1-8b-chat::default","modelScoreConfigSlug":"qwen1-5-1-8b-chat","modelScoreStateName":"Qwen1.5 1.8B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":839.3442,"rankLo":824.0,"rankHi":854.0},{"slug":"qwen2-5-0-5b","modelId":"qwen2-5-0-5b","modelSlug":"qwen2-5-0-5b","name":"Qwen2.5 0.5B","organizationSlug":"alibaba","n":14,"rankEligible":true,"score":78.13,"scoreLo":67.54,"scoreHi":86.87,"modelScoreStateId":"qwen2-5-0-5b::default","modelScoreConfigSlug":"qwen2-5-0-5b","modelScoreStateName":"Qwen2.5 0.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":839.3892,"rankLo":823.0,"rankHi":855.0},{"slug":"qwen1-5-1-8b","modelId":"qwen1-5-1-8b","modelSlug":"qwen1-5-1-8b","name":"Qwen1.5 1.8B","organizationSlug":"alibaba","n":15,"rankEligible":true,"score":78.01,"scoreLo":67.22,"scoreHi":86.85,"modelScoreStateId":"qwen1-5-1-8b::default","modelScoreConfigSlug":"qwen1-5-1-8b","modelScoreStateName":"Qwen1.5 1.8B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":839.7611,"rankLo":823.0,"rankHi":855.025},{"slug":"starcoder2-7b","modelId":"starcoder2-7b","modelSlug":"starcoder2-7b","name":"StarCoder2-7B","organizationSlug":"hugging-face","n":26,"rankEligible":true,"score":77.34,"scoreLo":67.12,"scoreHi":85.59,"modelScoreStateId":"starcoder2-7b::default","modelScoreConfigSlug":"starcoder2-7b","modelScoreStateName":"StarCoder2-7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":842.0403,"rankLo":829.0,"rankHi":855.0},{"slug":"gemini-nano-1","modelId":"gemini-nano-1","modelSlug":"gemini-nano-1","name":"Gemini Nano 1","organizationSlug":"google","n":11,"rankEligible":true,"score":76.99,"scoreLo":65.09,"scoreHi":86.59,"modelScoreStateId":"gemini-nano-1::default","modelScoreConfigSlug":"gemini-nano-1","modelScoreStateName":"Gemini Nano 1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":842.5767,"rankLo":821.0,"rankHi":861.0},{"slug":"gemma-1-1-2b-it","modelId":"gemma-1-1-2b-it","modelSlug":"gemma-1-1-2b-it","name":"Gemma 1.1 2B IT","organizationSlug":"google","n":9,"rankEligible":true,"score":76.98,"scoreLo":65.18,"scoreHi":86.84,"modelScoreStateId":"gemma-1-1-2b-it::default","modelScoreConfigSlug":"gemma-1-1-2b-it","modelScoreStateName":"Gemma 1.1 2B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":842.6344,"rankLo":821.0,"rankHi":860.0},{"slug":"chatglm2-6b","modelId":"chatglm2-6b","modelSlug":"chatglm2-6b","name":"ChatGLM2 6B (chat)","organizationSlug":"zhipu-ai","n":9,"rankEligible":true,"score":76.64,"scoreLo":65.55,"scoreHi":86.23,"modelScoreStateId":"chatglm2-6b::default","modelScoreConfigSlug":"chatglm2-6b","modelScoreStateName":"ChatGLM2 6B (chat)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":843.7836,"rankLo":826.0,"rankHi":860.0},{"slug":"deepseek-coder-6-7b-base","modelId":"deepseek-coder-6-7b-base","modelSlug":"deepseek-coder-6-7b-base","name":"DeepSeek-Coder-Base 6.7B","organizationSlug":"deepseek","n":22,"rankEligible":true,"score":75.66,"scoreLo":65.27,"scoreHi":84.38,"modelScoreStateId":"deepseek-coder-6-7b-base::default","modelScoreConfigSlug":"deepseek-coder-6-7b-base","modelScoreStateName":"DeepSeek-Coder-Base 6.7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":846.9619,"rankLo":834.0,"rankHi":859.0},{"slug":"stablelm-2-zephyr-1-6b","modelId":"stablelm-2-zephyr-1-6b","modelSlug":"stablelm-2-zephyr-1-6b","name":"StableLM 2 Zephyr 1.6B","organizationSlug":"stability-ai","n":13,"rankEligible":true,"score":75.65,"scoreLo":64.49,"scoreHi":84.95,"modelScoreStateId":"stablelm-2-zephyr-1-6b::default","modelScoreConfigSlug":"stablelm-2-zephyr-1-6b","modelScoreStateName":"StableLM 2 Zephyr 1.6B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":846.7139,"rankLo":829.0,"rankHi":861.0},{"slug":"qwen2-5-0-5b-instruct","modelId":"qwen2-5-0-5b-instruct","modelSlug":"qwen2-5-0-5b-instruct","name":"Qwen2.5 0.5B Instruct","organizationSlug":"alibaba","n":11,"rankEligible":true,"score":75.54,"scoreLo":63.86,"scoreHi":85.45,"modelScoreStateId":"qwen2-5-0-5b-instruct::default","modelScoreConfigSlug":"qwen2-5-0-5b-instruct","modelScoreStateName":"Qwen2.5 0.5B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":846.8308,"rankLo":827.0,"rankHi":863.0},{"slug":"qwen2-5-coder-0-5b","modelId":"qwen2-5-coder-0-5b","modelSlug":"qwen2-5-coder-0-5b","name":"Qwen2.5 Coder 0.5B","organizationSlug":"alibaba","n":14,"rankEligible":true,"score":75.36,"scoreLo":63.85,"scoreHi":84.92,"modelScoreStateId":"qwen2-5-coder-0-5b::default","modelScoreConfigSlug":"qwen2-5-coder-0-5b","modelScoreStateName":"Qwen2.5 Coder 0.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":847.5117,"rankLo":831.0,"rankHi":862.0},{"slug":"gpt-3-6-7b","modelId":"gpt-3-6-7b","modelSlug":"gpt-3-6-7b","name":"GPT-3 6.7B","organizationSlug":"openai","n":9,"rankEligible":true,"score":75.2,"scoreLo":63.27,"scoreHi":85.89,"modelScoreStateId":"gpt-3-6-7b::default","modelScoreConfigSlug":"gpt-3-6-7b","modelScoreStateName":"GPT-3 6.7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":847.3431,"rankLo":821.0,"rankHi":865.0},{"slug":"gemma-2b-it","modelId":"gemma-2b-it","modelSlug":"gemma-2b-it","name":"Gemma 2B IT","organizationSlug":"google","n":18,"rankEligible":true,"score":74.77,"scoreLo":64.46,"scoreHi":83.48,"modelScoreStateId":"gemma-2b-it::default","modelScoreConfigSlug":"gemma-2b-it","modelScoreStateName":"Gemma 2B IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":849.4767,"rankLo":837.0,"rankHi":861.0},{"slug":"gemma-2b","modelId":"gemma-2b","modelSlug":"gemma-2b","name":"Gemma 2B","organizationSlug":"google","n":40,"rankEligible":true,"score":74.75,"scoreLo":64.66,"scoreHi":82.9,"modelScoreStateId":"gemma-2b::default","modelScoreConfigSlug":"gemma-2b","modelScoreStateName":"Gemma 2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":849.7086,"rankLo":841.0,"rankHi":858.0},{"slug":"marin-8b-instruct","modelId":"marin-8b-instruct","modelSlug":"marin-8b-instruct","name":"Marin 8B Instruct","organizationSlug":"marin","n":5,"rankEligible":true,"score":74.53,"scoreLo":61.31,"scoreHi":85.66,"modelScoreStateId":"marin-8b-instruct::default","modelScoreConfigSlug":"marin-8b-instruct-helm-capabilities","modelScoreStateName":"Marin 8B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":848.9686,"rankLo":824.0,"rankHi":866.0},{"slug":"mpt-7b","modelId":"mpt-7b","modelSlug":"mpt-7b","name":"MPT-7B","organizationSlug":"mosaicml","n":26,"rankEligible":true,"score":74.42,"scoreLo":63.54,"scoreHi":83.47,"modelScoreStateId":"mpt-7b::default","modelScoreConfigSlug":"mpt-7b","modelScoreStateName":"MPT-7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":850.3564,"rankLo":838.0,"rankHi":861.0},{"slug":"vicuna-7b-v1-1","modelId":"vicuna-7b-v1-1","modelSlug":"vicuna-7b-v1-1","name":"Vicuna 7B v1.1","organizationSlug":"lmsys","n":4,"rankEligible":true,"score":73.59,"scoreLo":58.28,"scoreHi":86.67,"modelScoreStateId":"vicuna-7b-v1-1::default","modelScoreConfigSlug":"vicuna-7b-v1-1","modelScoreStateName":"Vicuna 7B v1.1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":850.0175,"rankLo":818.0,"rankHi":871.0},{"slug":"llama-7b","modelId":"llama-7b","modelSlug":"llama-7b","name":"LLaMA 7B","organizationSlug":"meta","n":18,"rankEligible":true,"score":73.44,"scoreLo":62.17,"scoreHi":82.69,"modelScoreStateId":"llama-7b::default","modelScoreConfigSlug":"llama-7b","modelScoreStateName":"LLaMA 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":852.7453,"rankLo":839.0,"rankHi":863.0},{"slug":"deepseek-vl-1-3b-chat","modelId":"deepseek-vl-1-3b-chat","modelSlug":"deepseek-vl-1-3b-chat","name":"DeepSeek-VL 1.3B Chat","organizationSlug":"deepseek","n":10,"rankEligible":true,"score":72.81,"scoreLo":60.77,"scoreHi":82.68,"modelScoreStateId":"deepseek-vl-1-3b-chat::default","modelScoreConfigSlug":"deepseek-vl-1-3b-chat","modelScoreStateName":"DeepSeek-VL 1.3B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":853.9636,"rankLo":838.0,"rankHi":866.0},{"slug":"qwen2-0-5b","modelId":"qwen2-0-5b","modelSlug":"qwen2-0-5b","name":"Qwen2-0.5B","organizationSlug":"alibaba","n":21,"rankEligible":true,"score":72.56,"scoreLo":61.39,"scoreHi":82.22,"modelScoreStateId":"qwen2-0-5b::default","modelScoreConfigSlug":"qwen2-0-5b","modelScoreStateName":"Qwen2-0.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":854.8264,"rankLo":841.975,"rankHi":865.0},{"slug":"open-llama-7b","modelId":"open-llama-7b","modelSlug":"open-llama-7b","name":"OpenLLaMA 7B","organizationSlug":"openlm-research","n":4,"rankEligible":true,"score":72.47,"scoreLo":58.7,"scoreHi":83.38,"modelScoreStateId":"open-llama-7b::default","modelScoreConfigSlug":"open-llama-7b","modelScoreStateName":"OpenLLaMA 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":853.9128,"rankLo":833.0,"rankHi":869.0},{"slug":"starcoder2-3b","modelId":"starcoder2-3b","modelSlug":"starcoder2-3b","name":"StarCoder2-3B","organizationSlug":"hugging-face","n":25,"rankEligible":true,"score":71.84,"scoreLo":60.46,"scoreHi":80.93,"modelScoreStateId":"starcoder2-3b::default","modelScoreConfigSlug":"starcoder2-3b","modelScoreStateName":"StarCoder2-3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":856.5022,"rankLo":845.0,"rankHi":865.0},{"slug":"recurrentgemma-2b","modelId":"recurrentgemma-2b","modelSlug":"recurrentgemma-2b","name":"RecurrentGemma 2B","organizationSlug":"google","n":28,"rankEligible":true,"score":71.5,"scoreLo":60.46,"scoreHi":80.34,"modelScoreStateId":"recurrentgemma-2b::default","modelScoreConfigSlug":"recurrentgemma-2b","modelScoreStateName":"RecurrentGemma 2B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":857.4539,"rankLo":848.0,"rankHi":864.0},{"slug":"redpajama-incite-7b-base","modelId":"redpajama-incite-7b-base","modelSlug":"redpajama-incite-7b-base","name":"RedPajama INCITE 7B Base","organizationSlug":"together-ai","n":11,"rankEligible":true,"score":69.73,"scoreLo":57.6,"scoreHi":80.0,"modelScoreStateId":"redpajama-incite-7b-base::default","modelScoreConfigSlug":"redpajama-incite-7b-base","modelScoreStateName":"RedPajama INCITE 7B Base","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":860.0675,"rankLo":847.0,"rankHi":869.0},{"slug":"gpt-3-2-7b","modelId":"gpt-3-2-7b","modelSlug":"gpt-3-2-7b","name":"GPT-3 2.7B","organizationSlug":"openai","n":9,"rankEligible":true,"score":69.68,"scoreLo":55.83,"scoreHi":81.44,"modelScoreStateId":"gpt-3-2-7b::default","modelScoreConfigSlug":"gpt-3-2-7b","modelScoreStateName":"GPT-3 2.7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":859.4628,"rankLo":841.0,"rankHi":871.0},{"slug":"deepseek-coder-1-3b-instruct","modelId":"deepseek-coder-1-3b-instruct","modelSlug":"deepseek-coder-1-3b-instruct","name":"DeepSeek-Coder-Instruct 1.3B","organizationSlug":"deepseek","n":12,"rankEligible":true,"score":67.97,"scoreLo":53.96,"scoreHi":79.32,"modelScoreStateId":"deepseek-coder-1-3b-instruct::default","modelScoreConfigSlug":"deepseek-coder-1-3b-instruct","modelScoreStateName":"DeepSeek-Coder-Instruct 1.3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":862.5967,"rankLo":849.0,"rankHi":873.0},{"slug":"gpt-neox-20b","modelId":"gpt-neox-20b","modelSlug":"gpt-neox-20b","name":"GPT-NeoX-20B","organizationSlug":"eleutherai","n":12,"rankEligible":true,"score":67.82,"scoreLo":55.2,"scoreHi":77.98,"modelScoreStateId":"gpt-neox-20b::default","modelScoreConfigSlug":"gpt-neox-20b","modelScoreStateName":"GPT-NeoX-20B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":863.1642,"rankLo":853.0,"rankHi":871.0},{"slug":"dolly-v2-12b","modelId":"dolly-v2-12b","modelSlug":"dolly-v2-12b","name":"Dolly 2.0 12B","organizationSlug":"databricks","n":11,"rankEligible":true,"score":67.13,"scoreLo":54.55,"scoreHi":77.25,"modelScoreStateId":"dolly-v2-12b::default","modelScoreConfigSlug":"dolly-v2-12b","modelScoreStateName":"Dolly 2.0 12B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":864.1594,"rankLo":855.0,"rankHi":871.0},{"slug":"qwen1-5-0-5b","modelId":"qwen1-5-0-5b","modelSlug":"qwen1-5-0-5b","name":"Qwen1.5 0.5B","organizationSlug":"alibaba","n":13,"rankEligible":true,"score":65.8,"scoreLo":52.41,"scoreHi":76.7,"modelScoreStateId":"qwen1-5-0-5b::default","modelScoreConfigSlug":"qwen1-5-0-5b","modelScoreStateName":"Qwen1.5 0.5B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":865.76,"rankLo":856.0,"rankHi":874.0},{"slug":"qwen2-0-5b-instruct","modelId":"qwen2-0-5b-instruct","modelSlug":"qwen2-0-5b-instruct","name":"Qwen2 0.5B Instruct","organizationSlug":"alibaba","n":8,"rankEligible":true,"score":63.33,"scoreLo":48.47,"scoreHi":75.68,"modelScoreStateId":"qwen2-0-5b-instruct::default","modelScoreConfigSlug":"qwen2-0-5b-instruct","modelScoreStateName":"Qwen2 0.5B Instruct","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":868.4878,"rankLo":858.0,"rankHi":878.0},{"slug":"intellect-1","modelId":"intellect-1","modelSlug":"intellect-1","name":"INTELLECT-1","organizationSlug":"prime-intellect","n":6,"rankEligible":true,"score":62.87,"scoreLo":46.19,"scoreHi":76.84,"modelScoreStateId":"intellect-1::default","modelScoreConfigSlug":"intellect-1","modelScoreStateName":"INTELLECT-1","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":868.6717,"rankLo":854.0,"rankHi":879.025},{"slug":"llama-3-2-1b","modelId":"llama-3-2-1b","modelSlug":"llama-3-2-1b","name":"Llama 3.2 1B","organizationSlug":"meta","n":6,"rankEligible":true,"score":62.61,"scoreLo":47.8,"scoreHi":75.63,"modelScoreStateId":"llama-3-2-1b::default","modelScoreConfigSlug":"llama-3-2-1b","modelScoreStateName":"Llama 3.2 1B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":869.2053,"rankLo":858.0,"rankHi":879.0},{"slug":"gemma-3-1b-pt","modelId":"gemma-3-1b-pt","modelSlug":"gemma-3-1b-pt","name":"Gemma 3 1B PT","organizationSlug":"google","n":11,"rankEligible":true,"score":62.32,"scoreLo":47.72,"scoreHi":74.09,"modelScoreStateId":"gemma-3-1b-pt::default","modelScoreConfigSlug":"gemma-3-1b-pt","modelScoreStateName":"Gemma 3 1B PT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":869.7122,"rankLo":861.0,"rankHi":877.025},{"slug":"falcon-7b","modelId":"falcon-7b","modelSlug":"falcon-7b","name":"Falcon 7B","organizationSlug":"tii","n":37,"rankEligible":true,"score":61.22,"scoreLo":48.59,"scoreHi":71.36,"modelScoreStateId":"falcon-7b::default","modelScoreConfigSlug":"falcon-7b","modelScoreStateName":"Falcon 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":871.0675,"rankLo":866.0,"rankHi":877.0},{"slug":"gpt-3-xl","modelId":"gpt-3-xl","modelSlug":"gpt-3-xl","name":"GPT-3 XL","organizationSlug":"openai","n":10,"rankEligible":true,"score":61.16,"scoreLo":45.97,"scoreHi":73.63,"modelScoreStateId":"gpt-3-xl::default","modelScoreConfigSlug":"gpt-3-xl","modelScoreStateName":"GPT-3 XL","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":870.8692,"rankLo":862.0,"rankHi":879.0},{"slug":"cerebras-gpt-13b","modelId":"cerebras-gpt-13b","modelSlug":"cerebras-gpt-13b","name":"Cerebras-GPT 13B","organizationSlug":"cerebras","n":4,"rankEligible":true,"score":59.94,"scoreLo":42.95,"scoreHi":73.79,"modelScoreStateId":"cerebras-gpt-13b::default","modelScoreConfigSlug":"cerebras-gpt-13b","modelScoreStateName":"Cerebras-GPT 13B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":871.8861,"rankLo":861.0,"rankHi":881.0},{"slug":"qwen1-5-0-5b-chat","modelId":"qwen1-5-0-5b-chat","modelSlug":"qwen1-5-0-5b-chat","name":"Qwen1.5 0.5B Chat","organizationSlug":"alibaba","n":9,"rankEligible":true,"score":59.21,"scoreLo":43.96,"scoreHi":71.77,"modelScoreStateId":"qwen1-5-0-5b-chat::default","modelScoreConfigSlug":"qwen1-5-0-5b-chat","modelScoreStateName":"Qwen1.5 0.5B Chat","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":872.8247,"rankLo":864.0,"rankHi":880.0},{"slug":"deepseek-coder-1-3b-base","modelId":"deepseek-coder-1-3b-base","modelSlug":"deepseek-coder-1-3b-base","name":"DeepSeek-Coder-Base 1.3B","organizationSlug":"deepseek","n":14,"rankEligible":true,"score":57.87,"scoreLo":43.02,"scoreHi":69.64,"modelScoreStateId":"deepseek-coder-1-3b-base::default","modelScoreConfigSlug":"deepseek-coder-1-3b-base","modelScoreStateName":"DeepSeek-Coder-Base 1.3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":874.1683,"rankLo":866.0,"rankHi":881.0},{"slug":"opt-1-3b","modelId":"opt-1-3b","modelSlug":"opt-1-3b","name":"OPT-1.3B","organizationSlug":"meta","n":16,"rankEligible":true,"score":55.83,"scoreLo":40.91,"scoreHi":68.61,"modelScoreStateId":"opt-1-3b::default","modelScoreConfigSlug":"opt-1-3b","modelScoreStateName":"OPT-1.3B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":875.9225,"rankLo":869.0,"rankHi":882.0},{"slug":"gpt-3-large","modelId":"gpt-3-large","modelSlug":"gpt-3-large","name":"GPT-3 Large","organizationSlug":"openai","n":9,"rankEligible":true,"score":55.42,"scoreLo":39.97,"scoreHi":69.12,"modelScoreStateId":"gpt-3-large::default","modelScoreConfigSlug":"gpt-3-large","modelScoreStateName":"GPT-3 Large","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":876.1547,"rankLo":868.0,"rankHi":882.0},{"slug":"stablelm-tuned-alpha-7b","modelId":"stablelm-tuned-alpha-7b","modelSlug":"stablelm-tuned-alpha-7b","name":"StableLM Tuned Alpha 7B","organizationSlug":"stability-ai","n":5,"rankEligible":true,"score":54.29,"scoreLo":39.02,"scoreHi":67.51,"modelScoreStateId":"stablelm-tuned-alpha-7b::default","modelScoreConfigSlug":"stablelm-tuned-alpha-7b","modelScoreStateName":"StableLM Tuned Alpha 7B","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":877.07,"rankLo":870.0,"rankHi":883.0},{"slug":"gpt-2-1558m","modelId":"gpt-2-1558m","modelSlug":"gpt-2-1558m","name":"GPT-2 (1.5B)","organizationSlug":"openai","n":21,"rankEligible":true,"score":53.01,"scoreLo":37.34,"scoreHi":66.47,"modelScoreStateId":"gpt-2-1558m::default","modelScoreConfigSlug":"gpt-2-1558m","modelScoreStateName":"GPT-2 (1.5B)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":878.0106,"rankLo":872.0,"rankHi":883.0},{"slug":"gpt-2-774m","modelId":"gpt-2-774m","modelSlug":"gpt-2-774m","name":"GPT-2 (774M)","organizationSlug":"openai","n":6,"rankEligible":true,"score":49.22,"scoreLo":31.54,"scoreHi":63.52,"modelScoreStateId":"gpt-2-774m::default","modelScoreConfigSlug":"gpt-2-774m","modelScoreStateName":"GPT-2 (774M)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":880.1822,"rankLo":873.0,"rankHi":884.0},{"slug":"gemma-3-270m-it","modelId":"gemma-3-270m-it","modelSlug":"gemma-3-270m-it","name":"Gemma 3 270M IT","organizationSlug":"google","n":6,"rankEligible":true,"score":49.2,"scoreLo":28.71,"scoreHi":67.22,"modelScoreStateId":"gemma-3-270m-it::default","modelScoreConfigSlug":"gemma-3-270m-it","modelScoreStateName":"Gemma 3 270M IT","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":879.5881,"rankLo":868.0,"rankHi":885.0},{"slug":"gemma-3-270m","modelId":"gemma-3-270m","modelSlug":"gemma-3-270m","name":"Gemma 3 270M","organizationSlug":"google","n":14,"rankEligible":true,"score":46.31,"scoreLo":27.66,"scoreHi":62.05,"modelScoreStateId":"gemma-3-270m::default","modelScoreConfigSlug":"gemma-3-270m","modelScoreStateName":"Gemma 3 270M","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":881.4578,"rankLo":875.0,"rankHi":885.0},{"slug":"gpt-3-medium","modelId":"gpt-3-medium","modelSlug":"gpt-3-medium","name":"GPT-3 Medium","organizationSlug":"openai","n":10,"rankEligible":true,"score":45.39,"scoreLo":28.77,"scoreHi":59.23,"modelScoreStateId":"gpt-3-medium::default","modelScoreConfigSlug":"gpt-3-medium","modelScoreStateName":"GPT-3 Medium","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":882.0978,"rankLo":878.0,"rankHi":885.0},{"slug":"gpt-2-355m","modelId":"gpt-2-355m","modelSlug":"gpt-2-355m","name":"GPT-2 (355M)","organizationSlug":"openai","n":4,"rankEligible":true,"score":44.67,"scoreLo":23.48,"scoreHi":60.8,"modelScoreStateId":"gpt-2-355m::default","modelScoreConfigSlug":"gpt-2-355m","modelScoreStateName":"GPT-2 (355M)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":882.0236,"rankLo":876.0,"rankHi":885.0},{"slug":"gpt-2-124m","modelId":"gpt-2-124m","modelSlug":"gpt-2-124m","name":"GPT-2 (124M)","organizationSlug":"openai","n":15,"rankEligible":true,"score":33.54,"scoreLo":13.25,"scoreHi":50.79,"modelScoreStateId":"gpt-2-124m::default","modelScoreConfigSlug":"gpt-2-124m","modelScoreStateName":"GPT-2 (124M)","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":884.7172,"rankLo":883.0,"rankHi":886.0},{"slug":"gpt-3-small","modelId":"gpt-3-small","modelSlug":"gpt-3-small","name":"GPT-3 Small","organizationSlug":"openai","n":11,"rankEligible":true,"score":18.7,"scoreLo":-3.79,"scoreHi":36.35,"modelScoreStateId":"gpt-3-small::default","modelScoreConfigSlug":"gpt-3-small","modelScoreStateName":"GPT-3 Small","modelScoreStateBestProbability":1.0,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none","rankMean":885.9711,"rankLo":885.0,"rankHi":886.0},{"slug":"gpt-5-3-codex-spark","modelId":"gpt-5-3-codex-spark","modelSlug":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark","organizationSlug":"openai","n":0,"rankEligible":false,"score":null,"scoreLo":null,"scoreHi":null,"modelScoreStateId":null,"modelScoreConfigSlug":null,"modelScoreStateName":null,"modelScoreStateBestProbability":null,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none"},{"slug":"laguna-xs-2-1","modelId":"laguna-xs-2-1","modelSlug":"laguna-xs-2-1","name":"Laguna XS 2.1","organizationSlug":"poolside","n":0,"rankEligible":false,"score":null,"scoreLo":null,"scoreHi":null,"modelScoreStateId":null,"modelScoreConfigSlug":null,"modelScoreStateName":null,"modelScoreStateBestProbability":null,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none"},{"slug":"nex-n2-mini","modelId":"nex-n2-mini","modelSlug":"nex-n2-mini","name":"Nex-N2-mini","organizationSlug":"nex-agi","n":0,"rankEligible":false,"score":null,"scoreLo":null,"scoreHi":null,"modelScoreStateId":null,"modelScoreConfigSlug":null,"modelScoreStateName":null,"modelScoreStateBestProbability":null,"modelScoreReasoningCondition":"default","modelScoreReasoningEffort":null,"modelScoreHarness":"none"}],"states":[{"stateId":"a-x-k2::enabled","configSlug":"a-x-k2-thinking","modelId":"a-x-k2","modelSlug":"a-x-k2","name":"A.X-K2 (thinking)","modelName":"A.X-K2","organizationSlug":"sk-telecom","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":146.62,"scoreLo":143.24,"scoreHi":150.11,"reasoningOffset":1.07,"rankEligible":true,"rankMean":168.6208,"rankLo":138.0,"rankHi":201.0},{"stateId":"agnes-2-5-pro-beta::default","configSlug":"agnes-2-5-pro-beta","modelId":"agnes-2-5-pro-beta","modelSlug":"agnes-2-5-pro-beta","name":"Agnes 2.5 Pro Beta","modelName":"Agnes 2.5 Pro Beta","organizationSlug":"sapiens-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":157.89,"scoreLo":154.03,"scoreHi":162.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":67.4411,"rankLo":44.0,"rankHi":84.0},{"stateId":"apertus-70b-instruct-2509::default","configSlug":"apertus-70b-instruct-2509","modelId":"apertus-70b-instruct-2509","modelSlug":"apertus-70b-instruct-2509","name":"Apertus 70B Instruct","modelName":"Apertus 70B Instruct","organizationSlug":"swiss-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":115.64,"scoreLo":103.51,"scoreHi":125.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":553.5856,"rankLo":432.0,"rankHi":682.0},{"stateId":"apertus-8b-instruct-2509::default","configSlug":"apertus-8b-instruct-2509","modelId":"apertus-8b-instruct-2509","modelSlug":"apertus-8b-instruct-2509","name":"Apertus 8B Instruct","modelName":"Apertus 8B Instruct","organizationSlug":"swiss-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":107.62,"scoreLo":94.16,"scoreHi":118.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":644.23,"rankLo":529.0,"rankHi":760.0},{"stateId":"apodex-1-1::default","configSlug":"apodex-1-1","modelId":"apodex-1-1","modelSlug":"apodex-1-1","name":"Apodex 1.1","modelName":"Apodex 1.1","organizationSlug":"apodex","reasoningSupport":"unknown","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":154.18,"scoreLo":150.9,"scoreHi":157.93,"reasoningOffset":0.0,"rankEligible":true,"rankMean":91.8547,"rankLo":78.0,"rankHi":111.0},{"stateId":"apriel-1-5-15b-thinker::default","configSlug":"apriel-1-5-15b-thinker","modelId":"apriel-1-5-15b-thinker","modelSlug":"apriel-1-5-15b-thinker","name":"Apriel-1.5-15B-Thinker","modelName":"Apriel-1.5-15B-Thinker","organizationSlug":"servicenow","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":134.2,"scoreLo":130.14,"scoreHi":138.22,"reasoningOffset":2.4,"rankEligible":true,"rankMean":318.9336,"rankLo":267.0,"rankHi":369.0},{"stateId":"apriel-1-6-15b-thinker::default","configSlug":"apriel-1-6-15b-thinker","modelId":"apriel-1-6-15b-thinker","modelSlug":"apriel-1-6-15b-thinker","name":"Apriel-1.6-15B-Thinker","modelName":"Apriel-1.6-15B-Thinker","organizationSlug":"servicenow","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":140.93,"scoreLo":136.32,"scoreHi":145.36,"reasoningOffset":2.4,"rankEligible":true,"rankMean":230.1147,"rankLo":185.0,"rankHi":287.0},{"stateId":"aria::default","configSlug":"aria","modelId":"aria","modelSlug":"aria","name":"Aria","modelName":"Aria","organizationSlug":"rhymes-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":114.5,"scoreLo":109.32,"scoreHi":119.13,"reasoningOffset":0.0,"rankEligible":true,"rankMean":568.1206,"rankLo":524.0,"rankHi":618.0},{"stateId":"atria-dawn-preview::default","configSlug":"atria-dawn-preview-bfcl-v4","modelId":"atria-dawn-preview","modelSlug":"atria-dawn-preview","name":"Atria Dawn Preview","modelName":"Atria Dawn Preview","organizationSlug":"shanghai-ai-lab","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":6,"score":160.79,"scoreLo":156.49,"scoreHi":165.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":41.31,"rankLo":18.0,"rankHi":69.0},{"stateId":"atria-dawn-preview::effort:high","configSlug":"atria-dawn-preview-high-claude-code-2-1-119","modelId":"atria-dawn-preview","modelSlug":"atria-dawn-preview","name":"Atria Dawn Preview (high)","modelName":"Atria Dawn Preview","organizationSlug":"shanghai-ai-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":161.32,"scoreLo":156.38,"scoreHi":166.86,"reasoningOffset":0.53,"rankEligible":false},{"stateId":"aya-23-35b::default","configSlug":"aya-23-35b","modelId":"aya-23-35b","modelSlug":"aya-23-35b","name":"Aya-23-35B","modelName":"Aya-23-35B","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":103.6,"scoreLo":96.06,"scoreHi":110.06,"reasoningOffset":0.0,"rankEligible":true,"rankMean":688.1925,"rankLo":643.0,"rankHi":730.0},{"stateId":"aya-expanse-32b::default","configSlug":"aya-expanse-32b","modelId":"aya-expanse-32b","modelSlug":"aya-expanse-32b","name":"Aya Expanse 32B","modelName":"Aya Expanse 32B","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":114.56,"scoreLo":109.22,"scoreHi":119.22,"reasoningOffset":0.0,"rankEligible":true,"rankMean":567.4819,"rankLo":523.975,"rankHi":617.0},{"stateId":"baichuan2-13b-base::default","configSlug":"baichuan2-13b-base","modelId":"baichuan2-13b-base","modelSlug":"baichuan2-13b-base","name":"Baichuan2-13B-Base","modelName":"Baichuan2-13B-Base","organizationSlug":"baichuan","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":88.46,"scoreLo":79.32,"scoreHi":96.09,"reasoningOffset":0.0,"rankEligible":true,"rankMean":797.8225,"rankLo":771.0,"rankHi":820.025},{"stateId":"baichuan2-13b-chat::default","configSlug":"baichuan2-13b-chat","modelId":"baichuan2-13b-chat","modelSlug":"baichuan2-13b-chat","name":"Baichuan2-13B-Chat","modelName":"Baichuan2-13B-Chat","organizationSlug":"baichuan","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":93.43,"scoreLo":84.44,"scoreHi":101.22,"reasoningOffset":0.0,"rankEligible":true,"rankMean":767.9856,"rankLo":723.975,"rankHi":805.0},{"stateId":"baichuan2-7b-base::default","configSlug":"baichuan2-7b-base","modelId":"baichuan2-7b-base","modelSlug":"baichuan2-7b-base","name":"Baichuan2-7B-Base","modelName":"Baichuan2-7B-Base","organizationSlug":"baichuan","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":80.72,"scoreLo":70.62,"scoreHi":89.24,"reasoningOffset":0.0,"rankEligible":true,"rankMean":830.5428,"rankLo":810.0,"rankHi":849.025},{"stateId":"beam-1-0-501b-a23b::default","configSlug":"beam-1-0-501b-a23b","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B","modelName":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":34,"score":154.78,"scoreLo":152.0,"scoreHi":158.03,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"beam-1-0-501b-a23b::effort:high","configSlug":"beam-1-0-501b-a23b-high","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B (high)","modelName":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":4,"score":153.57,"scoreLo":150.85,"scoreHi":156.79,"reasoningOffset":-1.21,"rankEligible":true},{"stateId":"beam-1-0-501b-a23b::effort:low","configSlug":"beam-1-0-501b-a23b-low","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B (low)","modelName":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":3,"score":150.72,"scoreLo":148.14,"scoreHi":153.6,"reasoningOffset":-4.06,"rankEligible":false},{"stateId":"beam-1-0-501b-a23b::effort:max","configSlug":"beam-1-0-501b-a23b-max","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B (max)","modelName":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":23,"score":155.21,"scoreLo":152.2,"scoreHi":158.77,"reasoningOffset":0.43,"rankEligible":true,"rankMean":83.8028,"rankLo":76.0,"rankHi":95.0},{"stateId":"beam-1-0-501b-a23b::effort:medium","configSlug":"beam-1-0-501b-a23b-medium","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B (medium)","modelName":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":4,"score":151.71,"scoreLo":149.16,"scoreHi":154.65,"reasoningOffset":-3.07,"rankEligible":true},{"stateId":"beam-1-0-501b-a23b::effort:minimal","configSlug":"beam-1-0-501b-a23b-minimal-chisel","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B (minimal)","modelName":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":149.82,"scoreLo":145.04,"scoreHi":154.84,"reasoningOffset":-4.96,"rankEligible":false},{"stateId":"beam-1-0-501b-a23b::effort:xhigh","configSlug":"beam-1-0-501b-a23b-xhigh","modelId":"beam-1-0-501b-a23b","modelSlug":"beam-1-0-501b-a23b","name":"Beam 1.0 501B A23B (xhigh)","modelName":"Beam 1.0 501B A23B","organizationSlug":"reflection-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":3,"score":153.99,"scoreLo":151.16,"scoreHi":157.31,"reasoningOffset":-0.79,"rankEligible":false},{"stateId":"c4ai-command-a-03-2025::default","configSlug":"c4ai-command-a-03-2025","modelId":"c4ai-command-a-03-2025","modelSlug":"c4ai-command-a-03-2025","name":"Command A","modelName":"Command A","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":123.99,"scoreLo":120.57,"scoreHi":127.03,"reasoningOffset":0.0,"rankEligible":true,"rankMean":448.4797,"rankLo":417.0,"rankHi":483.0},{"stateId":"c4ai-command-r-08-2024::default","configSlug":"c4ai-command-r-08-2024","modelId":"c4ai-command-r-08-2024","modelSlug":"c4ai-command-r-08-2024","name":"C4AI Command R 08-2024","modelName":"C4AI Command R 08-2024","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":117.03,"scoreLo":108.48,"scoreHi":123.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":537.8975,"rankLo":455.0,"rankHi":629.0},{"stateId":"c4ai-command-r-plus::default","configSlug":"c4ai-command-r-plus","modelId":"c4ai-command-r-plus","modelSlug":"c4ai-command-r-plus","name":"C4AI Command R+","modelName":"C4AI Command R+","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":34,"score":110.33,"scoreLo":105.49,"scoreHi":114.24,"reasoningOffset":0.0,"rankEligible":true,"rankMean":619.5461,"rankLo":591.975,"rankHi":647.0},{"stateId":"c4ai-command-r-plus-08-2024::default","configSlug":"c4ai-command-r-plus-08-2024","modelId":"c4ai-command-r-plus-08-2024","modelSlug":"c4ai-command-r-plus-08-2024","name":"C4AI Command R+ 08-2024","modelName":"C4AI Command R+ 08-2024","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":114.76,"scoreLo":109.44,"scoreHi":119.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":565.0544,"rankLo":522.0,"rankHi":614.0},{"stateId":"c4ai-command-r-v01::default","configSlug":"c4ai-command-r-v01","modelId":"c4ai-command-r-v01","modelSlug":"c4ai-command-r-v01","name":"Command R","modelName":"Command R","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":102.54,"scoreLo":96.53,"scoreHi":107.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":698.2225,"rankLo":675.0,"rankHi":720.0},{"stateId":"c4ai-command-r7b-12-2024::default","configSlug":"c4ai-command-r7b-12-2024","modelId":"c4ai-command-r7b-12-2024","modelSlug":"c4ai-command-r7b-12-2024","name":"Command R7B (Dec 2024)","modelName":"Command R7B (Dec 2024)","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":109.2,"scoreLo":103.21,"scoreHi":114.67,"reasoningOffset":0.0,"rankEligible":true,"rankMean":631.9994,"rankLo":581.0,"rankHi":679.0},{"stateId":"celeris-1::default","configSlug":"celeris-1","modelId":"celeris-1","modelSlug":"celeris-1","name":"Celeris-1","modelName":"Celeris-1","organizationSlug":"celeris","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":131.65,"scoreLo":127.77,"scoreHi":135.02,"reasoningOffset":0.0,"rankEligible":true,"rankMean":352.95,"rankLo":311.0,"rankHi":399.0},{"stateId":"cerebras-gpt-13b::default","configSlug":"cerebras-gpt-13b","modelId":"cerebras-gpt-13b","modelSlug":"cerebras-gpt-13b","name":"Cerebras-GPT 13B","modelName":"Cerebras-GPT 13B","organizationSlug":"cerebras","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":59.94,"scoreLo":42.95,"scoreHi":73.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":871.8861,"rankLo":861.0,"rankHi":881.0},{"stateId":"chatglm2-6b::default","configSlug":"chatglm2-6b","modelId":"chatglm2-6b","modelSlug":"chatglm2-6b","name":"ChatGLM2 6B (chat)","modelName":"ChatGLM2 6B (chat)","organizationSlug":"zhipu-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":76.64,"scoreLo":65.55,"scoreHi":86.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":843.7836,"rankLo":826.0,"rankHi":860.0},{"stateId":"chinchilla::default","configSlug":"chinchilla","modelId":"chinchilla","modelSlug":"chinchilla","name":"Chinchilla","modelName":"Chinchilla","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":103.95,"scoreLo":96.17,"scoreHi":111.03,"reasoningOffset":0.0,"rankEligible":true,"rankMean":684.6092,"rankLo":624.0,"rankHi":735.0},{"stateId":"claude-1-3::default","configSlug":"claude-1-3","modelId":"claude-1-3","modelSlug":"claude-1-3","name":"Claude 1.3","modelName":"Claude 1.3","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":116.94,"scoreLo":111.27,"scoreHi":122.49,"reasoningOffset":0.0,"rankEligible":true,"rankMean":539.1006,"rankLo":475.0,"rankHi":593.0},{"stateId":"claude-2-0::default","configSlug":"claude-2-0","modelId":"claude-2-0","modelSlug":"claude-2-0","name":"Claude 2","modelName":"Claude 2","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":112.95,"scoreLo":107.68,"scoreHi":117.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":587.1733,"rankLo":545.0,"rankHi":629.0},{"stateId":"claude-2-1::default","configSlug":"claude-2-1","modelId":"claude-2-1","modelSlug":"claude-2-1","name":"Claude 2.1","modelName":"Claude 2.1","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":113.97,"scoreLo":109.07,"scoreHi":118.3,"reasoningOffset":0.0,"rankEligible":true,"rankMean":574.5003,"rankLo":536.0,"rankHi":617.0},{"stateId":"claude-3-5-haiku-20241022::default","configSlug":"claude-3-5-haiku-20241022","modelId":"claude-3-5-haiku-20241022","modelSlug":"claude-3-5-haiku-20241022","name":"Haiku 3.5","modelName":"Haiku 3.5","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":70,"score":124.59,"scoreLo":121.84,"scoreHi":127.03,"reasoningOffset":0.0,"rankEligible":true,"rankMean":440.71,"rankLo":419.0,"rankHi":464.0},{"stateId":"claude-3-5-sonnet-20240620::default","configSlug":"claude-3-5-sonnet-20240620","modelId":"claude-3-5-sonnet-20240620","modelSlug":"claude-3-5-sonnet-20240620","name":"Claude 3.5 Sonnet (June 2024)","modelName":"Claude 3.5 Sonnet (June 2024)","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":72,"score":130.0,"scoreLo":130.0,"scoreHi":130.0,"reasoningOffset":0.0,"rankEligible":true,"rankMean":375.3594,"rankLo":354.0,"rankHi":397.0},{"stateId":"claude-3-5-sonnet-20241022::default","configSlug":"claude-3-5-sonnet-20241022","modelId":"claude-3-5-sonnet-20241022","modelSlug":"claude-3-5-sonnet-20241022","name":"Claude 3.5 Sonnet (Oct 2024)","modelName":"Claude 3.5 Sonnet (Oct 2024)","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":89,"score":133.56,"scoreLo":131.61,"scoreHi":135.45,"reasoningOffset":0.0,"rankEligible":true,"rankMean":328.2636,"rankLo":306.0,"rankHi":347.0},{"stateId":"claude-3-7-sonnet-20250219::default","configSlug":"claude-3-7-sonnet-20250219","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":52,"score":140.37,"scoreLo":138.61,"scoreHi":142.28,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-3-7-sonnet-20250219::effort:16k","configSlug":"claude-3-7-sonnet-20250219-16k","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (16k)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":8,"score":140.74,"scoreLo":138.89,"scoreHi":142.69,"reasoningOffset":0.37,"rankEligible":true,"rankMean":230.0469,"rankLo":215.0,"rankHi":249.0},{"stateId":"claude-3-7-sonnet-20250219::effort:1k","configSlug":"claude-3-7-sonnet-20250219-1k","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (1k)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":3,"score":138.72,"scoreLo":136.6,"scoreHi":140.88,"reasoningOffset":-1.65,"rankEligible":false},{"stateId":"claude-3-7-sonnet-20250219::effort:2500","configSlug":"claude-3-7-sonnet-20250219-2500","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (2500)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"2500","reasoningCondition":"effort:2500","isDefault":false,"n":1,"score":140.57,"scoreLo":135.87,"scoreHi":145.32,"reasoningOffset":0.2,"rankEligible":false},{"stateId":"claude-3-7-sonnet-20250219::effort:32k","configSlug":"claude-3-7-sonnet-20250219-32k","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (32k)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":2,"score":140.44,"scoreLo":138.21,"scoreHi":142.91,"reasoningOffset":0.07,"rankEligible":false},{"stateId":"claude-3-7-sonnet-20250219::effort:64k","configSlug":"claude-3-7-sonnet-20250219-64k","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (64k)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":7,"score":139.49,"scoreLo":137.53,"scoreHi":141.59,"reasoningOffset":-0.88,"rankEligible":true},{"stateId":"claude-3-7-sonnet-20250219::effort:8k","configSlug":"claude-3-7-sonnet-20250219-8k","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (8k)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":2,"score":140.13,"scoreLo":138.05,"scoreHi":142.3,"reasoningOffset":-0.25,"rankEligible":false},{"stateId":"claude-3-7-sonnet-20250219::enabled","configSlug":"claude-3-7-sonnet-20250219-thinking","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (thinking)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":34,"score":138.55,"scoreLo":136.76,"scoreHi":140.38,"reasoningOffset":-1.83,"rankEligible":true},{"stateId":"claude-3-haiku-20240307::default","configSlug":"claude-3-haiku-20240307","modelId":"claude-3-haiku-20240307","modelSlug":"claude-3-haiku-20240307","name":"Claude 3 Haiku","modelName":"Claude 3 Haiku","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":61,"score":111.98,"scoreLo":107.81,"scoreHi":115.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":599.4311,"rankLo":578.0,"rankHi":621.0},{"stateId":"claude-3-opus-20240229::default","configSlug":"claude-3-opus-20240229","modelId":"claude-3-opus-20240229","modelSlug":"claude-3-opus-20240229","name":"Opus 3","modelName":"Opus 3","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":66,"score":123.66,"scoreLo":120.98,"scoreHi":126.09,"reasoningOffset":0.0,"rankEligible":true,"rankMean":452.5603,"rankLo":432.0,"rankHi":474.0},{"stateId":"claude-3-sonnet-20240229::default","configSlug":"claude-3-sonnet-20240229","modelId":"claude-3-sonnet-20240229","modelSlug":"claude-3-sonnet-20240229","name":"Claude 3 Sonnet","modelName":"Claude 3 Sonnet","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":50,"score":117.5,"scoreLo":113.89,"scoreHi":120.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":533.03,"rankLo":513.0,"rankHi":552.0},{"stateId":"claude-fable-5::enabled","configSlug":"claude-fable-5","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (enabled)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":true,"n":36,"score":165.42,"scoreLo":161.48,"scoreHi":170.2,"reasoningOffset":1.63,"rankEligible":true,"rankMean":12.2692,"rankLo":9.0,"rankHi":16.0},{"stateId":"claude-fable-5-1::default","configSlug":"claude-fable-5-1","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1","modelName":"Claude Fable 5.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":58,"score":166.06,"scoreLo":162.18,"scoreHi":170.7,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-fable-5-1::enabled","configSlug":"claude-fable-5-1-concept-synth-cascade","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1 (enabled)","modelName":"Claude Fable 5.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":166.08,"scoreLo":161.68,"scoreHi":171.27,"reasoningOffset":0.02,"rankEligible":false},{"stateId":"claude-fable-5-1::effort:high","configSlug":"claude-fable-5-1-high","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1 (high)","modelName":"Claude Fable 5.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":72,"score":165.59,"scoreLo":161.76,"scoreHi":170.24,"reasoningOffset":-0.46,"rankEligible":true},{"stateId":"claude-fable-5-1::effort:low","configSlug":"claude-fable-5-1-low","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1 (low)","modelName":"Claude Fable 5.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":23,"score":163.17,"scoreLo":159.55,"scoreHi":167.57,"reasoningOffset":-2.89,"rankEligible":true},{"stateId":"claude-fable-5-1::effort:max","configSlug":"claude-fable-5-1-max","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1 (max)","modelName":"Claude Fable 5.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":121,"score":166.81,"scoreLo":162.9,"scoreHi":171.55,"reasoningOffset":0.76,"rankEligible":true,"rankMean":7.8703,"rankLo":6.0,"rankHi":9.0},{"stateId":"claude-fable-5-1::effort:medium","configSlug":"claude-fable-5-1-medium","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1 (medium)","modelName":"Claude Fable 5.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":30,"score":164.24,"scoreLo":160.46,"scoreHi":168.75,"reasoningOffset":-1.81,"rankEligible":true},{"stateId":"claude-fable-5-1::effort:xhigh","configSlug":"claude-fable-5-1-xhigh","modelId":"claude-fable-5-1","modelSlug":"claude-fable-5-1","name":"Claude Fable 5.1 (xhigh)","modelName":"Claude Fable 5.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":33,"score":166.09,"scoreLo":162.12,"scoreHi":170.8,"reasoningOffset":0.03,"rankEligible":true},{"stateId":"claude-fable-5::effort:adaptive","configSlug":"claude-fable-5-adaptive-claude-code","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (adaptive)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"adaptive","reasoningCondition":"effort:adaptive","isDefault":false,"n":1,"score":161.98,"scoreLo":155.1,"scoreHi":168.61,"reasoningOffset":-1.81,"rankEligible":false},{"stateId":"claude-fable-5::effort:high","configSlug":"claude-fable-5-high","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (high)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":63,"score":163.86,"scoreLo":160.2,"scoreHi":168.19,"reasoningOffset":0.07,"rankEligible":true},{"stateId":"claude-fable-5::effort:low","configSlug":"claude-fable-5-low","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (low)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":27,"score":162.09,"scoreLo":158.53,"scoreHi":166.4,"reasoningOffset":-1.71,"rankEligible":true},{"stateId":"claude-fable-5::effort:max","configSlug":"claude-fable-5-max","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (max)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":122,"score":164.16,"scoreLo":160.47,"scoreHi":168.55,"reasoningOffset":0.37,"rankEligible":true},{"stateId":"claude-fable-5::effort:medium","configSlug":"claude-fable-5-medium","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (medium)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":23,"score":163.28,"scoreLo":159.61,"scoreHi":167.73,"reasoningOffset":-0.51,"rankEligible":true},{"stateId":"claude-fable-5::effort:minimal","configSlug":"claude-fable-5-minimal","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (minimal)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":159.59,"scoreLo":154.21,"scoreHi":165.58,"reasoningOffset":-4.21,"rankEligible":false},{"stateId":"claude-fable-5::disabled","configSlug":"claude-fable-5-no-thinking-policybench-forced-answer","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (disabled)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"default","reasoningCondition":"disabled","isDefault":false,"n":1,"score":159.75,"scoreLo":152.58,"scoreHi":167.35,"reasoningOffset":-4.05,"rankEligible":false},{"stateId":"claude-fable-5::default","configSlug":"claude-fable-5-unspecified","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":163.79,"scoreLo":160.17,"scoreHi":168.17,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-fable-5::effort:xhigh","configSlug":"claude-fable-5-xhigh","modelId":"claude-fable-5","modelSlug":"claude-fable-5","name":"Claude Fable 5 (xhigh)","modelName":"Claude Fable 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":32,"score":164.58,"scoreLo":160.84,"scoreHi":169.1,"reasoningOffset":0.78,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::default","configSlug":"claude-haiku-4-5-20251001","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":57,"score":143.11,"scoreLo":141.34,"scoreHi":145.05,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::effort:128k","configSlug":"claude-haiku-4-5-20251001-128k-launch-post","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (128k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"128k","reasoningCondition":"effort:128k","isDefault":false,"n":9,"score":142.86,"scoreLo":140.6,"scoreHi":145.38,"reasoningOffset":-0.25,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::effort:16k","configSlug":"claude-haiku-4-5-20251001-16k","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (16k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":7,"score":143.38,"scoreLo":141.49,"scoreHi":145.55,"reasoningOffset":0.28,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::effort:1k","configSlug":"claude-haiku-4-5-20251001-1k","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (1k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":5,"score":141.38,"scoreLo":139.41,"scoreHi":143.46,"reasoningOffset":-1.72,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::effort:2k","configSlug":"claude-haiku-4-5-20251001-2k","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (2k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"2k","reasoningCondition":"effort:2k","isDefault":false,"n":1,"score":144.64,"scoreLo":140.51,"scoreHi":149.38,"reasoningOffset":1.54,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:32k","configSlug":"claude-haiku-4-5-20251001-32k","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (32k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":14,"score":142.89,"scoreLo":141.09,"scoreHi":144.87,"reasoningOffset":-0.21,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::effort:4k","configSlug":"claude-haiku-4-5-20251001-4k-sab","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (4k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"4k","reasoningCondition":"effort:4k","isDefault":false,"n":1,"score":143.61,"scoreLo":138.94,"scoreHi":148.37,"reasoningOffset":0.51,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:59k","configSlug":"claude-haiku-4-5-20251001-59k","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (59k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"59k","reasoningCondition":"effort:59k","isDefault":false,"n":1,"score":141.9,"scoreLo":138.59,"scoreHi":145.09,"reasoningOffset":-1.21,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:60000","configSlug":"claude-haiku-4-5-20251001-60000","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (60000)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"60000","reasoningCondition":"effort:60000","isDefault":false,"n":1,"score":142.68,"scoreLo":138.11,"scoreHi":147.09,"reasoningOffset":-0.43,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:64k","configSlug":"claude-haiku-4-5-20251001-64k","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (64k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":1,"score":141.91,"scoreLo":138.93,"scoreHi":144.85,"reasoningOffset":-1.2,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:8k","configSlug":"claude-haiku-4-5-20251001-8k","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (8k)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":3,"score":142.66,"scoreLo":140.69,"scoreHi":144.79,"reasoningOffset":-0.44,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:high","configSlug":"claude-haiku-4-5-20251001-high","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (high)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":10,"score":143.77,"scoreLo":141.34,"scoreHi":146.34,"reasoningOffset":0.66,"rankEligible":true,"rankMean":198.9622,"rankLo":178.0,"rankHi":217.0},{"stateId":"claude-haiku-4-5-20251001::effort:low","configSlug":"claude-haiku-4-5-20251001-low-futuresearch-drb","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (low)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":141.11,"scoreLo":137.0,"scoreHi":145.28,"reasoningOffset":-2.0,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:medium","configSlug":"claude-haiku-4-5-20251001-medium","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (medium)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":3,"score":141.78,"scoreLo":137.98,"scoreHi":145.69,"reasoningOffset":-1.33,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::effort:minimal","configSlug":"claude-haiku-4-5-20251001-minimal","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (minimal)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":138.91,"scoreLo":134.36,"scoreHi":143.67,"reasoningOffset":-4.2,"rankEligible":false},{"stateId":"claude-haiku-4-5-20251001::disabled","configSlug":"claude-haiku-4-5-20251001-no-thinking","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (disabled)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":25,"score":139.33,"scoreLo":137.01,"scoreHi":141.71,"reasoningOffset":-3.77,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::enabled","configSlug":"claude-haiku-4-5-20251001-thinking","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (thinking)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":55,"score":142.6,"scoreLo":140.59,"scoreHi":144.81,"reasoningOffset":-0.51,"rankEligible":true},{"stateId":"claude-haiku-4-5-20251001::effort:xhigh","configSlug":"claude-haiku-4-5-20251001-xhigh-aikido-code-analysis","modelId":"claude-haiku-4-5-20251001","modelSlug":"claude-haiku-4-5-20251001","name":"Haiku 4.5 (xhigh)","modelName":"Haiku 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":143.81,"scoreLo":140.89,"scoreHi":146.99,"reasoningOffset":0.7,"rankEligible":false},{"stateId":"claude-haiku-5-5::default","configSlug":"claude-haiku-5-5","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":162.59,"scoreLo":159.09,"scoreHi":166.72,"reasoningOffset":0.0,"rankEligible":true,"rankMean":23.2561,"rankLo":19.0,"rankHi":29.0},{"stateId":"claude-haiku-5-5::effort:high","configSlug":"claude-haiku-5-5-high","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5 (high)","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":32,"score":160.22,"scoreLo":156.97,"scoreHi":164.08,"reasoningOffset":-2.37,"rankEligible":true},{"stateId":"claude-haiku-5-5::effort:low","configSlug":"claude-haiku-5-5-low","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5 (low)","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":22,"score":156.14,"scoreLo":153.25,"scoreHi":159.52,"reasoningOffset":-6.46,"rankEligible":true},{"stateId":"claude-haiku-5-5::effort:max","configSlug":"claude-haiku-5-5-max","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5 (max)","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":91,"score":162.41,"scoreLo":158.94,"scoreHi":166.57,"reasoningOffset":-0.19,"rankEligible":true},{"stateId":"claude-haiku-5-5::effort:medium","configSlug":"claude-haiku-5-5-medium","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5 (medium)","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":23,"score":157.89,"scoreLo":154.79,"scoreHi":161.62,"reasoningOffset":-4.71,"rankEligible":true},{"stateId":"claude-haiku-5-5::disabled","configSlug":"claude-haiku-5-5-no-thinking","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5 (disabled)","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":4,"score":150.21,"scoreLo":145.89,"scoreHi":154.84,"reasoningOffset":-12.39,"rankEligible":true},{"stateId":"claude-haiku-5-5::enabled","configSlug":"claude-haiku-5-5-proximus","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5 (enabled)","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":1,"score":164.77,"scoreLo":160.17,"scoreHi":170.1,"reasoningOffset":2.18,"rankEligible":false},{"stateId":"claude-haiku-5-5::effort:xhigh","configSlug":"claude-haiku-5-5-xhigh","modelId":"claude-haiku-5-5","modelSlug":"claude-haiku-5-5","name":"Claude Haiku 5.5 (xhigh)","modelName":"Claude Haiku 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":24,"score":161.78,"scoreLo":158.45,"scoreHi":166.0,"reasoningOffset":-0.81,"rankEligible":true},{"stateId":"claude-instant-1-1::default","configSlug":"claude-instant-1-1","modelId":"claude-instant-1-1","modelSlug":"claude-instant-1-1","name":"Claude Instant 1.1","modelName":"Claude Instant 1.1","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":111.89,"scoreLo":105.46,"scoreHi":117.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":600.1764,"rankLo":540.975,"rankHi":656.0},{"stateId":"claude-instant-1-2::default","configSlug":"claude-instant-1-2","modelId":"claude-instant-1-2","modelSlug":"claude-instant-1-2","name":"Claude Instant 1.2","modelName":"Claude Instant 1.2","organizationSlug":"anthropic","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":109.54,"scoreLo":102.97,"scoreHi":115.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":627.9581,"rankLo":571.0,"rankHi":683.0},{"stateId":"claude-mythos-preview::default","configSlug":"claude-mythos-preview-anthropic-agent","modelId":"claude-mythos-preview","modelSlug":"claude-mythos-preview","name":"Claude Mythos Preview","modelName":"Claude Mythos Preview","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":1,"score":162.8,"scoreLo":158.72,"scoreHi":167.68,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"claude-mythos-preview::enabled","configSlug":"claude-mythos-preview-caisi-math-prompt-no-system-safeguards","modelId":"claude-mythos-preview","modelSlug":"claude-mythos-preview","name":"Claude Mythos Preview (enabled)","modelName":"Claude Mythos Preview","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":5,"score":163.7,"scoreLo":159.08,"scoreHi":168.87,"reasoningOffset":0.89,"rankEligible":true,"rankMean":19.8339,"rankLo":9.0,"rankHi":40.0},{"stateId":"claude-mythos-preview::effort:max","configSlug":"claude-mythos-preview-max","modelId":"claude-mythos-preview","modelSlug":"claude-mythos-preview","name":"Claude Mythos Preview (max)","modelName":"Claude Mythos Preview","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":19,"score":163.57,"scoreLo":159.49,"scoreHi":168.55,"reasoningOffset":0.77,"rankEligible":true},{"stateId":"claude-mythos-preview::disabled","configSlug":"claude-mythos-preview-no-thinking","modelId":"claude-mythos-preview","modelSlug":"claude-mythos-preview","name":"Claude Mythos Preview (disabled)","modelName":"Claude Mythos Preview","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":158.52,"scoreLo":151.88,"scoreHi":165.94,"reasoningOffset":-4.29,"rankEligible":false},{"stateId":"claude-opus-4-1-20250805::default","configSlug":"claude-opus-4-1-20250805","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":42,"score":145.99,"scoreLo":143.85,"scoreHi":148.31,"reasoningOffset":0.0,"rankEligible":true,"rankMean":175.5003,"rankLo":162.0,"rankHi":189.0},{"stateId":"claude-opus-4-1-20250805::effort:16k","configSlug":"claude-opus-4-1-20250805-16k","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1 (16k)","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":6,"score":145.82,"scoreLo":143.43,"scoreHi":148.38,"reasoningOffset":-0.16,"rankEligible":true},{"stateId":"claude-opus-4-1-20250805::effort:24k","configSlug":"claude-opus-4-1-20250805-24k","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1 (24k)","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"24k","reasoningCondition":"effort:24k","isDefault":false,"n":1,"score":146.91,"scoreLo":142.79,"scoreHi":151.91,"reasoningOffset":0.92,"rankEligible":false},{"stateId":"claude-opus-4-1-20250805::effort:27k","configSlug":"claude-opus-4-1-20250805-27k","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1 (27k)","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"27k","reasoningCondition":"effort:27k","isDefault":false,"n":2,"score":145.03,"scoreLo":141.45,"scoreHi":148.78,"reasoningOffset":-0.96,"rankEligible":false},{"stateId":"claude-opus-4-1-20250805::effort:32k","configSlug":"claude-opus-4-1-20250805-32k","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1 (32k)","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":2,"score":145.48,"scoreLo":142.83,"scoreHi":148.23,"reasoningOffset":-0.51,"rankEligible":false},{"stateId":"claude-opus-4-1-20250805::effort:64k","configSlug":"claude-opus-4-1-20250805-64k","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1 (64k)","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":6,"score":145.13,"scoreLo":142.87,"scoreHi":147.69,"reasoningOffset":-0.85,"rankEligible":true},{"stateId":"claude-opus-4-1-20250805::disabled","configSlug":"claude-opus-4-1-20250805-no-thinking","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1 (disabled)","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":41,"score":142.22,"scoreLo":140.29,"scoreHi":144.24,"reasoningOffset":-3.77,"rankEligible":true},{"stateId":"claude-opus-4-1-20250805::enabled","configSlug":"claude-opus-4-1-20250805-thinking","modelId":"claude-opus-4-1-20250805","modelSlug":"claude-opus-4-1-20250805","name":"Opus 4.1 (thinking)","modelName":"Opus 4.1","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":57,"score":144.17,"scoreLo":142.26,"scoreHi":146.27,"reasoningOffset":-1.82,"rankEligible":true},{"stateId":"claude-opus-4-20250514::default","configSlug":"claude-opus-4-20250514","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":36,"score":144.22,"scoreLo":142.26,"scoreHi":146.3,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-opus-4-20250514::effort:10k","configSlug":"claude-opus-4-20250514-10k-helm-capabilities","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (10k)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"10k","reasoningCondition":"effort:10k","isDefault":false,"n":5,"score":144.47,"scoreLo":141.47,"scoreHi":147.89,"reasoningOffset":0.25,"rankEligible":true,"rankMean":191.5094,"rankLo":161.0,"rankHi":217.0},{"stateId":"claude-opus-4-20250514::effort:16k","configSlug":"claude-opus-4-20250514-16k","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (16k)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":8,"score":144.3,"scoreLo":142.23,"scoreHi":146.54,"reasoningOffset":0.08,"rankEligible":true},{"stateId":"claude-opus-4-20250514::effort:1k","configSlug":"claude-opus-4-20250514-1k","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (1k)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":4,"score":142.67,"scoreLo":140.48,"scoreHi":144.89,"reasoningOffset":-1.55,"rankEligible":true},{"stateId":"claude-opus-4-20250514::effort:31488","configSlug":"claude-opus-4-20250514-31488","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (31488)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"31488","reasoningCondition":"effort:31488","isDefault":false,"n":1,"score":142.72,"scoreLo":137.15,"scoreHi":148.21,"reasoningOffset":-1.5,"rankEligible":false},{"stateId":"claude-opus-4-20250514::effort:31k","configSlug":"claude-opus-4-20250514-31k","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (31k)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"31k","reasoningCondition":"effort:31k","isDefault":false,"n":5,"score":142.04,"scoreLo":139.45,"scoreHi":144.58,"reasoningOffset":-2.18,"rankEligible":true},{"stateId":"claude-opus-4-20250514::effort:32k","configSlug":"claude-opus-4-20250514-32k","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (32k)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":2,"score":143.84,"scoreLo":141.38,"scoreHi":146.4,"reasoningOffset":-0.38,"rankEligible":false},{"stateId":"claude-opus-4-20250514::effort:64k","configSlug":"claude-opus-4-20250514-64k","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (64k)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":6,"score":143.6,"scoreLo":141.44,"scoreHi":146.07,"reasoningOffset":-0.62,"rankEligible":true},{"stateId":"claude-opus-4-20250514::effort:8k","configSlug":"claude-opus-4-20250514-8k","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (8k)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":2,"score":143.86,"scoreLo":141.74,"scoreHi":146.11,"reasoningOffset":-0.36,"rankEligible":false},{"stateId":"claude-opus-4-20250514::effort:high","configSlug":"claude-opus-4-20250514-high","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (high)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":144.78,"scoreLo":141.77,"scoreHi":148.24,"reasoningOffset":0.56,"rankEligible":false},{"stateId":"claude-opus-4-20250514::disabled","configSlug":"claude-opus-4-20250514-no-thinking","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (disabled)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":51,"score":140.76,"scoreLo":138.99,"scoreHi":142.62,"reasoningOffset":-3.46,"rankEligible":true},{"stateId":"claude-opus-4-20250514::enabled","configSlug":"claude-opus-4-20250514-thinking","modelId":"claude-opus-4-20250514","modelSlug":"claude-opus-4-20250514","name":"Opus 4 (thinking)","modelName":"Opus 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":31,"score":143.8,"scoreLo":141.92,"scoreHi":145.9,"reasoningOffset":-0.43,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::default","configSlug":"claude-opus-4-5-20251101","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":117,"score":151.34,"scoreLo":148.98,"scoreHi":154.11,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::effort:128k","configSlug":"claude-opus-4-5-20251101-128k-terminus-2","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (128k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"128k","reasoningCondition":"effort:128k","isDefault":false,"n":1,"score":151.67,"scoreLo":147.81,"scoreHi":156.22,"reasoningOffset":0.32,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:16k","configSlug":"claude-opus-4-5-20251101-16k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (16k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":9,"score":151.63,"scoreLo":149.21,"scoreHi":154.48,"reasoningOffset":0.3,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::effort:1k","configSlug":"claude-opus-4-5-20251101-1k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (1k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":6,"score":149.23,"scoreLo":146.85,"scoreHi":151.92,"reasoningOffset":-2.11,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::effort:2k","configSlug":"claude-opus-4-5-20251101-2k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (2k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"2k","reasoningCondition":"effort:2k","isDefault":false,"n":1,"score":153.18,"scoreLo":148.94,"scoreHi":158.45,"reasoningOffset":1.84,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:31488","configSlug":"claude-opus-4-5-20251101-31488","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (31488)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"31488","reasoningCondition":"effort:31488","isDefault":false,"n":1,"score":150.13,"scoreLo":144.64,"scoreHi":155.82,"reasoningOffset":-1.21,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:32k","configSlug":"claude-opus-4-5-20251101-32k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (32k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":16,"score":151.29,"scoreLo":148.92,"scoreHi":154.0,"reasoningOffset":-0.04,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::effort:48k","configSlug":"claude-opus-4-5-20251101-48k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (48k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"48k","reasoningCondition":"effort:48k","isDefault":false,"n":1,"score":151.73,"scoreLo":146.73,"scoreHi":157.4,"reasoningOffset":0.4,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:59k","configSlug":"claude-opus-4-5-20251101-59k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (59k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"59k","reasoningCondition":"effort:59k","isDefault":false,"n":1,"score":149.97,"scoreLo":146.42,"scoreHi":153.7,"reasoningOffset":-1.37,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:60000","configSlug":"claude-opus-4-5-20251101-60000","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (60000)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"60000","reasoningCondition":"effort:60000","isDefault":false,"n":1,"score":150.82,"scoreLo":146.13,"scoreHi":155.44,"reasoningOffset":-0.52,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:64k","configSlug":"claude-opus-4-5-20251101-64k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (64k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":14,"score":152.3,"scoreLo":149.69,"scoreHi":155.38,"reasoningOffset":0.96,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::effort:65535","configSlug":"claude-opus-4-5-20251101-65535","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (65535)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"65535","reasoningCondition":"effort:65535","isDefault":false,"n":1,"score":151.24,"scoreLo":146.68,"scoreHi":156.14,"reasoningOffset":-0.09,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:8k","configSlug":"claude-opus-4-5-20251101-8k","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (8k)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":3,"score":150.85,"scoreLo":148.4,"scoreHi":153.63,"reasoningOffset":-0.49,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:high","configSlug":"claude-opus-4-5-20251101-high","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (high)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":30,"score":153.02,"scoreLo":150.31,"scoreHi":156.19,"reasoningOffset":1.68,"rankEligible":true,"rankMean":101.7483,"rankLo":90.0,"rankHi":113.0},{"stateId":"claude-opus-4-5-20251101::effort:low","configSlug":"claude-opus-4-5-20251101-low-futuresearch-drb","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (low)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":149.65,"scoreLo":145.28,"scoreHi":154.61,"reasoningOffset":-1.7,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::effort:minimal","configSlug":"claude-opus-4-5-20251101-minimal","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (minimal)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":147.08,"scoreLo":142.57,"scoreHi":152.12,"reasoningOffset":-4.26,"rankEligible":false},{"stateId":"claude-opus-4-5-20251101::disabled","configSlug":"claude-opus-4-5-20251101-no-thinking","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (disabled)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":25,"score":147.89,"scoreLo":145.57,"scoreHi":150.62,"reasoningOffset":-3.45,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::disabled:high","configSlug":"claude-opus-4-5-20251101-no-thinking-high","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (disabled, high)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"high","reasoningCondition":"disabled:high","isDefault":false,"n":17,"score":148.6,"scoreLo":146.0,"scoreHi":151.68,"reasoningOffset":-2.74,"rankEligible":true},{"stateId":"claude-opus-4-5-20251101::enabled","configSlug":"claude-opus-4-5-20251101-thinking","modelId":"claude-opus-4-5-20251101","modelSlug":"claude-opus-4-5-20251101","name":"Opus 4.5 (thinking)","modelName":"Opus 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":43,"score":151.14,"scoreLo":148.73,"scoreHi":153.91,"reasoningOffset":-0.2,"rankEligible":true},{"stateId":"claude-opus-4-6::default","configSlug":"claude-opus-4-6","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":67,"score":155.74,"scoreLo":152.94,"scoreHi":158.99,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-opus-4-6::effort:120k","configSlug":"claude-opus-4-6-120k","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (120k)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"120k","reasoningCondition":"effort:120k","isDefault":false,"n":1,"score":153.69,"scoreLo":149.43,"scoreHi":158.18,"reasoningOffset":-2.05,"rankEligible":false},{"stateId":"claude-opus-4-6::effort:123904","configSlug":"claude-opus-4-6-123904","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (123904)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"123904","reasoningCondition":"effort:123904","isDefault":false,"n":1,"score":156.28,"scoreLo":151.22,"scoreHi":162.22,"reasoningOffset":0.54,"rankEligible":false},{"stateId":"claude-opus-4-6::effort:16k","configSlug":"claude-opus-4-6-16k","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (16k)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":2,"score":156.04,"scoreLo":153.0,"scoreHi":159.67,"reasoningOffset":0.3,"rankEligible":false},{"stateId":"claude-opus-4-6::effort:2k","configSlug":"claude-opus-4-6-2k","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (2k)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"2k","reasoningCondition":"effort:2k","isDefault":false,"n":1,"score":157.38,"scoreLo":152.28,"scoreHi":163.11,"reasoningOffset":1.65,"rankEligible":false},{"stateId":"claude-opus-4-6::effort:32k","configSlug":"claude-opus-4-6-32k","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (32k)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":3,"score":155.35,"scoreLo":152.33,"scoreHi":158.9,"reasoningOffset":-0.39,"rankEligible":false},{"stateId":"claude-opus-4-6::effort:64k","configSlug":"claude-opus-4-6-64k","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (64k)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":10,"score":154.21,"scoreLo":151.52,"scoreHi":157.47,"reasoningOffset":-1.53,"rankEligible":true},{"stateId":"claude-opus-4-6::effort:8k","configSlug":"claude-opus-4-6-8k","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (8k)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":1,"score":155.41,"scoreLo":152.4,"scoreHi":159.11,"reasoningOffset":-0.33,"rankEligible":false},{"stateId":"claude-opus-4-6::effort:high","configSlug":"claude-opus-4-6-high","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (high)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":35,"score":157.33,"scoreLo":154.36,"scoreHi":160.84,"reasoningOffset":1.6,"rankEligible":true,"rankMean":72.5303,"rankLo":67.0,"rankHi":77.0},{"stateId":"claude-opus-4-6::effort:low","configSlug":"claude-opus-4-6-low","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (low)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":9,"score":155.68,"scoreLo":152.7,"scoreHi":159.22,"reasoningOffset":-0.06,"rankEligible":true},{"stateId":"claude-opus-4-6::effort:max","configSlug":"claude-opus-4-6-max","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (max)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":118,"score":156.12,"scoreLo":153.38,"scoreHi":159.44,"reasoningOffset":0.38,"rankEligible":true},{"stateId":"claude-opus-4-6::effort:medium","configSlug":"claude-opus-4-6-medium","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (medium)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":9,"score":156.94,"scoreLo":153.92,"scoreHi":160.62,"reasoningOffset":1.2,"rankEligible":true},{"stateId":"claude-opus-4-6::effort:minimal","configSlug":"claude-opus-4-6-minimal","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (minimal)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":151.08,"scoreLo":146.32,"scoreHi":156.27,"reasoningOffset":-4.65,"rankEligible":false},{"stateId":"claude-opus-4-6::disabled","configSlug":"claude-opus-4-6-no-thinking","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (disabled)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":21,"score":152.71,"scoreLo":149.76,"scoreHi":156.14,"reasoningOffset":-3.03,"rankEligible":true},{"stateId":"claude-opus-4-6::disabled:high","configSlug":"claude-opus-4-6-no-thinking-high","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (disabled, high)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"high","reasoningCondition":"disabled:high","isDefault":false,"n":9,"score":151.85,"scoreLo":148.85,"scoreHi":155.15,"reasoningOffset":-3.89,"rankEligible":true},{"stateId":"claude-opus-4-6::disabled:low","configSlug":"claude-opus-4-6-no-thinking-low-effort-jonathan-chang-visual-acuity","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (disabled, low)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"low","reasoningCondition":"disabled:low","isDefault":false,"n":1,"score":152.89,"scoreLo":147.52,"scoreHi":159.86,"reasoningOffset":-2.86,"rankEligible":false},{"stateId":"claude-opus-4-6::disabled:max","configSlug":"claude-opus-4-6-no-thinking-max","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (disabled, max)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"max","reasoningCondition":"disabled:max","isDefault":false,"n":6,"score":154.28,"scoreLo":151.23,"scoreHi":157.88,"reasoningOffset":-1.46,"rankEligible":true},{"stateId":"claude-opus-4-6::enabled","configSlug":"claude-opus-4-6-thinking","modelId":"claude-opus-4-6","modelSlug":"claude-opus-4-6","name":"Opus 4.6 (thinking)","modelName":"Opus 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":10,"score":156.61,"scoreLo":153.23,"scoreHi":160.55,"reasoningOffset":0.87,"rankEligible":true},{"stateId":"claude-opus-4-7::default","configSlug":"claude-opus-4-7","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":82,"score":157.88,"scoreLo":154.89,"scoreHi":161.36,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-opus-4-7::effort:2k","configSlug":"claude-opus-4-7-2k","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (2k)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"2k","reasoningCondition":"effort:2k","isDefault":false,"n":1,"score":159.41,"scoreLo":153.96,"scoreHi":165.44,"reasoningOffset":1.53,"rankEligible":false},{"stateId":"claude-opus-4-7::effort:8k","configSlug":"claude-opus-4-7-8k","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (8k)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":1,"score":157.5,"scoreLo":154.21,"scoreHi":161.41,"reasoningOffset":-0.38,"rankEligible":false},{"stateId":"claude-opus-4-7::effort:64k","configSlug":"claude-opus-4-7-adaptive-64k","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (64k)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":7,"score":156.8,"scoreLo":153.83,"scoreHi":160.34,"reasoningOffset":-1.08,"rankEligible":true},{"stateId":"claude-opus-4-7::effort:high","configSlug":"claude-opus-4-7-high","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (high)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":43,"score":158.02,"scoreLo":154.96,"scoreHi":161.56,"reasoningOffset":0.13,"rankEligible":true},{"stateId":"claude-opus-4-7::effort:low","configSlug":"claude-opus-4-7-low","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (low)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":11,"score":157.28,"scoreLo":154.18,"scoreHi":160.87,"reasoningOffset":-0.61,"rankEligible":true},{"stateId":"claude-opus-4-7::effort:max","configSlug":"claude-opus-4-7-max","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (max)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":104,"score":158.69,"scoreLo":155.6,"scoreHi":162.34,"reasoningOffset":0.81,"rankEligible":true,"rankMean":63.0631,"rankLo":56.0,"rankHi":69.0},{"stateId":"claude-opus-4-7::effort:medium","configSlug":"claude-opus-4-7-medium","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (medium)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":13,"score":157.66,"scoreLo":154.64,"scoreHi":161.4,"reasoningOffset":-0.22,"rankEligible":true},{"stateId":"claude-opus-4-7::effort:minimal","configSlug":"claude-opus-4-7-minimal","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (minimal)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":153.72,"scoreLo":148.84,"scoreHi":159.29,"reasoningOffset":-4.17,"rankEligible":false},{"stateId":"claude-opus-4-7::disabled","configSlug":"claude-opus-4-7-no-thinking","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (disabled)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":11,"score":153.25,"scoreLo":149.84,"scoreHi":157.1,"reasoningOffset":-4.63,"rankEligible":true},{"stateId":"claude-opus-4-7::disabled:high","configSlug":"claude-opus-4-7-no-thinking-high","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (disabled, high)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"high","reasoningCondition":"disabled:high","isDefault":false,"n":8,"score":154.03,"scoreLo":150.76,"scoreHi":157.77,"reasoningOffset":-3.85,"rankEligible":true},{"stateId":"claude-opus-4-7::disabled:low","configSlug":"claude-opus-4-7-no-thinking-low-effort-jonathan-chang-visual-acuity","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (disabled, low)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"low","reasoningCondition":"disabled:low","isDefault":false,"n":1,"score":155.17,"scoreLo":149.74,"scoreHi":162.23,"reasoningOffset":-2.71,"rankEligible":false},{"stateId":"claude-opus-4-7::disabled:max","configSlug":"claude-opus-4-7-no-thinking-max-browsecomp-single-agent-10m","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (disabled, max)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"max","reasoningCondition":"disabled:max","isDefault":false,"n":1,"score":156.97,"scoreLo":153.22,"scoreHi":161.48,"reasoningOffset":-0.91,"rankEligible":false},{"stateId":"claude-opus-4-7::enabled","configSlug":"claude-opus-4-7-thinking","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (thinking)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":2,"score":157.68,"scoreLo":154.19,"scoreHi":161.86,"reasoningOffset":-0.2,"rankEligible":false},{"stateId":"claude-opus-4-7::effort:xhigh","configSlug":"claude-opus-4-7-xhigh","modelId":"claude-opus-4-7","modelSlug":"claude-opus-4-7","name":"Opus 4.7 (xhigh)","modelName":"Opus 4.7","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":21,"score":157.42,"scoreLo":154.37,"scoreHi":161.0,"reasoningOffset":-0.46,"rankEligible":true},{"stateId":"claude-opus-4-8::default","configSlug":"claude-opus-4-8","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":134,"score":160.0,"scoreLo":156.82,"scoreHi":163.83,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-opus-4-8::enabled","configSlug":"claude-opus-4-8-exploitbench-autonudge","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (enabled)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":1,"score":160.54,"scoreLo":156.43,"scoreHi":165.38,"reasoningOffset":0.54,"rankEligible":false},{"stateId":"claude-opus-4-8::effort:high","configSlug":"claude-opus-4-8-high","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (high)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":64,"score":159.32,"scoreLo":156.17,"scoreHi":163.02,"reasoningOffset":-0.68,"rankEligible":true},{"stateId":"claude-opus-4-8::effort:low","configSlug":"claude-opus-4-8-low","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (low)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":27,"score":157.59,"scoreLo":154.55,"scoreHi":161.19,"reasoningOffset":-2.41,"rankEligible":true},{"stateId":"claude-opus-4-8::effort:max","configSlug":"claude-opus-4-8-max","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (max)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":146,"score":160.89,"scoreLo":157.63,"scoreHi":164.82,"reasoningOffset":0.9,"rankEligible":true,"rankMean":39.3519,"rankLo":33.0,"rankHi":46.0},{"stateId":"claude-opus-4-8::effort:medium","configSlug":"claude-opus-4-8-medium","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (medium)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":30,"score":158.74,"scoreLo":155.67,"scoreHi":162.41,"reasoningOffset":-1.26,"rankEligible":true},{"stateId":"claude-opus-4-8::disabled","configSlug":"claude-opus-4-8-no-thinking","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (disabled)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":14,"score":155.8,"scoreLo":152.22,"scoreHi":160.1,"reasoningOffset":-4.2,"rankEligible":true},{"stateId":"claude-opus-4-8::disabled:low","configSlug":"claude-opus-4-8-no-thinking-low-effort-jonathan-chang-visual-acuity","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (disabled, low)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"low","reasoningCondition":"disabled:low","isDefault":false,"n":1,"score":157.29,"scoreLo":151.67,"scoreHi":164.72,"reasoningOffset":-2.72,"rankEligible":false},{"stateId":"claude-opus-4-8::effort:xhigh","configSlug":"claude-opus-4-8-xhigh","modelId":"claude-opus-4-8","modelSlug":"claude-opus-4-8","name":"Opus 4.8 (xhigh)","modelName":"Opus 4.8","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":37,"score":160.09,"scoreLo":156.89,"scoreHi":163.93,"reasoningOffset":0.1,"rankEligible":true},{"stateId":"claude-opus-5::default","configSlug":"claude-opus-5","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":94,"score":164.63,"scoreLo":160.95,"scoreHi":169.11,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-opus-5::effort:16k","configSlug":"claude-opus-5-16k","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (16k)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":1,"score":165.02,"scoreLo":161.21,"scoreHi":169.77,"reasoningOffset":0.4,"rankEligible":false},{"stateId":"claude-opus-5-5::default","configSlug":"claude-opus-5-5","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5","modelName":"Claude Opus 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":66,"score":167.15,"scoreLo":163.15,"scoreHi":171.97,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-opus-5-5::enabled","configSlug":"claude-opus-5-5-concept-synth-cascade","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5 (enabled)","modelName":"Claude Opus 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":168.49,"scoreLo":163.98,"scoreHi":174.09,"reasoningOffset":1.33,"rankEligible":false},{"stateId":"claude-opus-5-5::effort:high","configSlug":"claude-opus-5-5-high","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5 (high)","modelName":"Claude Opus 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":60,"score":167.38,"scoreLo":163.34,"scoreHi":172.2,"reasoningOffset":0.22,"rankEligible":true},{"stateId":"claude-opus-5-5::effort:low","configSlug":"claude-opus-5-5-low","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5 (low)","modelName":"Claude Opus 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":30,"score":162.57,"scoreLo":159.06,"scoreHi":166.89,"reasoningOffset":-4.58,"rankEligible":true},{"stateId":"claude-opus-5-5::effort:max","configSlug":"claude-opus-5-5-max","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5 (max)","modelName":"Claude Opus 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":125,"score":168.71,"scoreLo":164.52,"scoreHi":173.74,"reasoningOffset":1.56,"rankEligible":true,"rankMean":2.2569,"rankLo":1.0,"rankHi":4.0},{"stateId":"claude-opus-5-5::effort:medium","configSlug":"claude-opus-5-5-medium","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5 (medium)","modelName":"Claude Opus 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":40,"score":165.74,"scoreLo":161.9,"scoreHi":170.38,"reasoningOffset":-1.41,"rankEligible":true},{"stateId":"claude-opus-5-5::effort:xhigh","configSlug":"claude-opus-5-5-xhigh","modelId":"claude-opus-5-5","modelSlug":"claude-opus-5-5","name":"Claude Opus 5.5 (xhigh)","modelName":"Claude Opus 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":36,"score":168.08,"scoreLo":164.01,"scoreHi":173.12,"reasoningOffset":0.93,"rankEligible":true},{"stateId":"claude-opus-5::enabled","configSlug":"claude-opus-5-concept-synth-cascade","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (enabled)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":3,"score":165.34,"scoreLo":161.04,"scoreHi":170.55,"reasoningOffset":0.71,"rankEligible":false},{"stateId":"claude-opus-5::effort:high","configSlug":"claude-opus-5-high","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (high)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":74,"score":164.49,"scoreLo":160.84,"scoreHi":168.91,"reasoningOffset":-0.13,"rankEligible":true},{"stateId":"claude-opus-5::effort:low","configSlug":"claude-opus-5-low","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (low)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":41,"score":161.23,"scoreLo":157.68,"scoreHi":165.28,"reasoningOffset":-3.4,"rankEligible":true},{"stateId":"claude-opus-5::effort:max","configSlug":"claude-opus-5-max","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (max)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":142,"score":165.2,"scoreLo":161.43,"scoreHi":169.68,"reasoningOffset":0.57,"rankEligible":true,"rankMean":13.1725,"rankLo":11.0,"rankHi":16.0},{"stateId":"claude-opus-5::effort:medium","configSlug":"claude-opus-5-medium","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (medium)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":43,"score":163.02,"scoreLo":159.44,"scoreHi":167.3,"reasoningOffset":-1.61,"rankEligible":true},{"stateId":"claude-opus-5::disabled","configSlug":"claude-opus-5-no-thinking","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (disabled)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":2,"score":162.77,"scoreLo":156.31,"scoreHi":169.98,"reasoningOffset":-1.86,"rankEligible":false},{"stateId":"claude-opus-5::effort:xhigh","configSlug":"claude-opus-5-xhigh","modelId":"claude-opus-5","modelSlug":"claude-opus-5","name":"Claude Opus 5 (xhigh)","modelName":"Claude Opus 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":48,"score":164.61,"scoreLo":160.92,"scoreHi":169.15,"reasoningOffset":-0.02,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::default","configSlug":"claude-sonnet-4-20250514","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":70,"score":143.87,"scoreLo":142.11,"scoreHi":145.88,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::effort:10k","configSlug":"claude-sonnet-4-20250514-10k-helm-capabilities","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (10k)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"10k","reasoningCondition":"effort:10k","isDefault":false,"n":5,"score":143.84,"scoreLo":140.77,"scoreHi":147.19,"reasoningOffset":-0.02,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::effort:16k","configSlug":"claude-sonnet-4-20250514-16k","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (16k)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":11,"score":144.2,"scoreLo":142.3,"scoreHi":146.25,"reasoningOffset":0.33,"rankEligible":true,"rankMean":194.6669,"rankLo":183.0,"rankHi":207.0},{"stateId":"claude-sonnet-4-20250514::effort:1k","configSlug":"claude-sonnet-4-20250514-1k","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (1k)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":4,"score":142.39,"scoreLo":140.36,"scoreHi":144.6,"reasoningOffset":-1.48,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::effort:31488","configSlug":"claude-sonnet-4-20250514-31488","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (31488)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"31488","reasoningCondition":"effort:31488","isDefault":false,"n":1,"score":142.13,"scoreLo":136.18,"scoreHi":147.6,"reasoningOffset":-1.73,"rankEligible":false},{"stateId":"claude-sonnet-4-20250514::effort:32k","configSlug":"claude-sonnet-4-20250514-32k","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (32k)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":4,"score":143.51,"scoreLo":141.21,"scoreHi":145.92,"reasoningOffset":-0.35,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::effort:59k","configSlug":"claude-sonnet-4-20250514-59k","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (59k)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"59k","reasoningCondition":"effort:59k","isDefault":false,"n":2,"score":143.03,"scoreLo":140.27,"scoreHi":146.03,"reasoningOffset":-0.83,"rankEligible":false},{"stateId":"claude-sonnet-4-20250514::effort:64k","configSlug":"claude-sonnet-4-20250514-64k","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (64k)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":6,"score":142.8,"scoreLo":140.57,"scoreHi":145.09,"reasoningOffset":-1.07,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::effort:8k","configSlug":"claude-sonnet-4-20250514-8k","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (8k)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":2,"score":143.42,"scoreLo":141.4,"scoreHi":145.57,"reasoningOffset":-0.44,"rankEligible":false},{"stateId":"claude-sonnet-4-20250514::effort:high","configSlug":"claude-sonnet-4-20250514-high","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (high)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":144.86,"scoreLo":142.2,"scoreHi":147.7,"reasoningOffset":1.0,"rankEligible":false},{"stateId":"claude-sonnet-4-20250514::effort:low","configSlug":"claude-sonnet-4-20250514-low","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (low)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":142.88,"scoreLo":139.39,"scoreHi":146.76,"reasoningOffset":-0.99,"rankEligible":false},{"stateId":"claude-sonnet-4-20250514::effort:medium","configSlug":"claude-sonnet-4-20250514-medium","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (medium)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":144.14,"scoreLo":140.56,"scoreHi":147.95,"reasoningOffset":0.27,"rankEligible":false},{"stateId":"claude-sonnet-4-20250514::disabled","configSlug":"claude-sonnet-4-20250514-no-thinking","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (disabled)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":70,"score":139.78,"scoreLo":138.18,"scoreHi":141.49,"reasoningOffset":-4.09,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::enabled","configSlug":"claude-sonnet-4-20250514-thinking","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (thinking)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":62,"score":141.79,"scoreLo":140.14,"scoreHi":143.65,"reasoningOffset":-2.07,"rankEligible":true},{"stateId":"claude-sonnet-4-20250514::effort:xhigh","configSlug":"claude-sonnet-4-20250514-xhigh","modelId":"claude-sonnet-4-20250514","modelSlug":"claude-sonnet-4-20250514","name":"Sonnet 4 (xhigh)","modelName":"Sonnet 4","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":144.84,"scoreLo":142.07,"scoreHi":147.9,"reasoningOffset":0.98,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::default","configSlug":"claude-sonnet-4-5-20250929","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":103,"score":147.18,"scoreLo":145.17,"scoreHi":149.48,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-sonnet-4-5-20250929::effort:128k","configSlug":"claude-sonnet-4-5-20250929-128k","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (128k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"128k","reasoningCondition":"effort:128k","isDefault":false,"n":1,"score":147.21,"scoreLo":143.04,"scoreHi":151.77,"reasoningOffset":0.04,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:16k","configSlug":"claude-sonnet-4-5-20250929-16k","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (16k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":9,"score":147.16,"scoreLo":145.04,"scoreHi":149.62,"reasoningOffset":-0.02,"rankEligible":true},{"stateId":"claude-sonnet-4-5-20250929::effort:1k","configSlug":"claude-sonnet-4-5-20250929-1k","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (1k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":4,"score":145.41,"scoreLo":143.3,"scoreHi":147.75,"reasoningOffset":-1.77,"rankEligible":true},{"stateId":"claude-sonnet-4-5-20250929::effort:200k","configSlug":"claude-sonnet-4-5-20250929-200k-anthropic-swe-bench-agent","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (200k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"200k","reasoningCondition":"effort:200k","isDefault":false,"n":1,"score":147.72,"scoreLo":142.53,"scoreHi":153.69,"reasoningOffset":0.54,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:31488","configSlug":"claude-sonnet-4-5-20250929-31488","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (31488)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"31488","reasoningCondition":"effort:31488","isDefault":false,"n":1,"score":145.77,"scoreLo":140.24,"scoreHi":151.35,"reasoningOffset":-1.4,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:32k","configSlug":"claude-sonnet-4-5-20250929-32k","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (32k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":17,"score":147.44,"scoreLo":145.39,"scoreHi":149.8,"reasoningOffset":0.26,"rankEligible":true},{"stateId":"claude-sonnet-4-5-20250929::effort:48k","configSlug":"claude-sonnet-4-5-20250929-48k","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (48k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"48k","reasoningCondition":"effort:48k","isDefault":false,"n":1,"score":147.64,"scoreLo":142.74,"scoreHi":153.04,"reasoningOffset":0.47,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:59k","configSlug":"claude-sonnet-4-5-20250929-59k","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (59k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"59k","reasoningCondition":"effort:59k","isDefault":false,"n":5,"score":146.31,"scoreLo":143.71,"scoreHi":149.18,"reasoningOffset":-0.87,"rankEligible":true},{"stateId":"claude-sonnet-4-5-20250929::effort:60000","configSlug":"claude-sonnet-4-5-20250929-60000","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (60000)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"60000","reasoningCondition":"effort:60000","isDefault":false,"n":1,"score":147.25,"scoreLo":142.48,"scoreHi":151.78,"reasoningOffset":0.07,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:65535","configSlug":"claude-sonnet-4-5-20250929-65535","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (65535)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"65535","reasoningCondition":"effort:65535","isDefault":false,"n":1,"score":147.15,"scoreLo":142.5,"scoreHi":151.85,"reasoningOffset":-0.03,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:8k","configSlug":"claude-sonnet-4-5-20250929-8k","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (8k)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":2,"score":146.87,"scoreLo":144.76,"scoreHi":149.35,"reasoningOffset":-0.3,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:high","configSlug":"claude-sonnet-4-5-20250929-high","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (high)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":20,"score":147.45,"scoreLo":145.29,"scoreHi":149.85,"reasoningOffset":0.27,"rankEligible":true,"rankMean":159.8333,"rankLo":146.975,"rankHi":173.0},{"stateId":"claude-sonnet-4-5-20250929::effort:low","configSlug":"claude-sonnet-4-5-20250929-low","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (low)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":146.13,"scoreLo":142.52,"scoreHi":150.07,"reasoningOffset":-1.05,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:medium","configSlug":"claude-sonnet-4-5-20250929-medium","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (medium)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":147.37,"scoreLo":143.83,"scoreHi":151.47,"reasoningOffset":0.19,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::effort:minimal","configSlug":"claude-sonnet-4-5-20250929-minimal","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (minimal)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":142.93,"scoreLo":138.49,"scoreHi":147.76,"reasoningOffset":-4.26,"rankEligible":false},{"stateId":"claude-sonnet-4-5-20250929::disabled","configSlug":"claude-sonnet-4-5-20250929-no-thinking","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (disabled)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":28,"score":143.28,"scoreLo":141.08,"scoreHi":145.62,"reasoningOffset":-3.9,"rankEligible":true},{"stateId":"claude-sonnet-4-5-20250929::enabled","configSlug":"claude-sonnet-4-5-20250929-thinking","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (thinking)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":62,"score":146.64,"scoreLo":144.73,"scoreHi":148.9,"reasoningOffset":-0.53,"rankEligible":true},{"stateId":"claude-sonnet-4-5-20250929::effort:xhigh","configSlug":"claude-sonnet-4-5-20250929-xhigh","modelId":"claude-sonnet-4-5-20250929","modelSlug":"claude-sonnet-4-5-20250929","name":"Sonnet 4.5 (xhigh)","modelName":"Sonnet 4.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":148.13,"scoreLo":145.09,"scoreHi":151.72,"reasoningOffset":0.96,"rankEligible":false},{"stateId":"claude-sonnet-4-6::default","configSlug":"claude-sonnet-4-6","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":68,"score":154.58,"scoreLo":151.98,"scoreHi":157.61,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-sonnet-4-6::effort:123904","configSlug":"claude-sonnet-4-6-123904","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (123904)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"123904","reasoningCondition":"effort:123904","isDefault":false,"n":1,"score":154.33,"scoreLo":149.01,"scoreHi":159.61,"reasoningOffset":-0.24,"rankEligible":false},{"stateId":"claude-sonnet-4-6::effort:16k","configSlug":"claude-sonnet-4-6-16k","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (16k)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":2,"score":154.68,"scoreLo":151.73,"scoreHi":158.2,"reasoningOffset":0.1,"rankEligible":false},{"stateId":"claude-sonnet-4-6::effort:2k","configSlug":"claude-sonnet-4-6-2k","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (2k)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"2k","reasoningCondition":"effort:2k","isDefault":false,"n":1,"score":156.25,"scoreLo":151.21,"scoreHi":162.23,"reasoningOffset":1.67,"rankEligible":false},{"stateId":"claude-sonnet-4-6::effort:31488","configSlug":"claude-sonnet-4-6-31488","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (31488)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"31488","reasoningCondition":"effort:31488","isDefault":false,"n":1,"score":153.32,"scoreLo":147.8,"scoreHi":159.43,"reasoningOffset":-1.26,"rankEligible":false},{"stateId":"claude-sonnet-4-6::effort:32k","configSlug":"claude-sonnet-4-6-32k","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (32k)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":4,"score":154.05,"scoreLo":151.14,"scoreHi":157.45,"reasoningOffset":-0.53,"rankEligible":true},{"stateId":"claude-sonnet-4-6::effort:8k","configSlug":"claude-sonnet-4-6-8k","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (8k)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":1,"score":154.25,"scoreLo":151.23,"scoreHi":157.72,"reasoningOffset":-0.32,"rankEligible":false},{"stateId":"claude-sonnet-4-6::effort:64k","configSlug":"claude-sonnet-4-6-adaptive-64k","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (64k)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":7,"score":153.19,"scoreLo":150.56,"scoreHi":156.33,"reasoningOffset":-1.38,"rankEligible":true},{"stateId":"claude-sonnet-4-6::effort:high","configSlug":"claude-sonnet-4-6-high","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (high)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":51,"score":154.67,"scoreLo":152.01,"scoreHi":157.76,"reasoningOffset":0.1,"rankEligible":true},{"stateId":"claude-sonnet-4-6::effort:low","configSlug":"claude-sonnet-4-6-low","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (low)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":12,"score":150.02,"scoreLo":146.93,"scoreHi":153.55,"reasoningOffset":-4.56,"rankEligible":true},{"stateId":"claude-sonnet-4-6::effort:max","configSlug":"claude-sonnet-4-6-max","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (max)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":88,"score":155.26,"scoreLo":152.51,"scoreHi":158.5,"reasoningOffset":0.68,"rankEligible":true,"rankMean":83.0925,"rankLo":79.0,"rankHi":89.0},{"stateId":"claude-sonnet-4-6::effort:medium","configSlug":"claude-sonnet-4-6-medium","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (medium)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":20,"score":153.6,"scoreLo":150.75,"scoreHi":156.9,"reasoningOffset":-0.98,"rankEligible":true},{"stateId":"claude-sonnet-4-6::effort:minimal","configSlug":"claude-sonnet-4-6-minimal","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (minimal)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":150.46,"scoreLo":145.79,"scoreHi":155.73,"reasoningOffset":-4.12,"rankEligible":false},{"stateId":"claude-sonnet-4-6::disabled","configSlug":"claude-sonnet-4-6-no-thinking","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (disabled)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":16,"score":152.17,"scoreLo":148.93,"scoreHi":155.82,"reasoningOffset":-2.41,"rankEligible":true},{"stateId":"claude-sonnet-4-6::disabled:high","configSlug":"claude-sonnet-4-6-no-thinking-high","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (disabled, high)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"high","reasoningCondition":"disabled:high","isDefault":false,"n":8,"score":150.03,"scoreLo":146.92,"scoreHi":153.33,"reasoningOffset":-4.55,"rankEligible":true},{"stateId":"claude-sonnet-4-6::disabled:low","configSlug":"claude-sonnet-4-6-no-thinking-low","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (disabled, low)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"low","reasoningCondition":"disabled:low","isDefault":false,"n":9,"score":150.01,"scoreLo":146.64,"scoreHi":153.57,"reasoningOffset":-4.57,"rankEligible":true},{"stateId":"claude-sonnet-4-6::disabled:max","configSlug":"claude-sonnet-4-6-no-thinking-max-browsecomp-single-agent-10m-system-card","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (disabled, max)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"max","reasoningCondition":"disabled:max","isDefault":false,"n":3,"score":153.58,"scoreLo":150.21,"scoreHi":157.4,"reasoningOffset":-1.0,"rankEligible":false},{"stateId":"claude-sonnet-4-6::enabled","configSlug":"claude-sonnet-4-6-thinking","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (thinking)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":6,"score":154.15,"scoreLo":150.96,"scoreHi":157.99,"reasoningOffset":-0.42,"rankEligible":true},{"stateId":"claude-sonnet-4-6::effort:xhigh","configSlug":"claude-sonnet-4-6-xhigh-claude-code","modelId":"claude-sonnet-4-6","modelSlug":"claude-sonnet-4-6","name":"Sonnet 4.6 (xhigh)","modelName":"Sonnet 4.6","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":155.09,"scoreLo":151.67,"scoreHi":158.91,"reasoningOffset":0.52,"rankEligible":false},{"stateId":"claude-sonnet-5::default","configSlug":"claude-sonnet-5","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":65,"score":158.89,"scoreLo":155.79,"scoreHi":162.62,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-sonnet-5::enabled","configSlug":"claude-sonnet-5-16k-forecastbench-zero-shot-with-crowd-forecast","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (enabled)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":3,"score":158.73,"scoreLo":154.82,"scoreHi":163.21,"reasoningOffset":-0.15,"rankEligible":false},{"stateId":"claude-sonnet-5-5::default","configSlug":"claude-sonnet-5-5","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","modelName":"Claude Sonnet 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":37,"score":166.71,"scoreLo":162.76,"scoreHi":171.49,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"claude-sonnet-5-5::enabled","configSlug":"claude-sonnet-5-5-concept-synth-cascade","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5 (enabled)","modelName":"Claude Sonnet 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":167.49,"scoreLo":163.17,"scoreHi":172.8,"reasoningOffset":0.78,"rankEligible":false},{"stateId":"claude-sonnet-5-5::effort:high","configSlug":"claude-sonnet-5-5-high","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5 (high)","modelName":"Claude Sonnet 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":37,"score":164.26,"scoreLo":160.54,"scoreHi":168.77,"reasoningOffset":-2.45,"rankEligible":true},{"stateId":"claude-sonnet-5-5::effort:low","configSlug":"claude-sonnet-5-5-low","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5 (low)","modelName":"Claude Sonnet 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":28,"score":159.32,"scoreLo":156.1,"scoreHi":163.18,"reasoningOffset":-7.39,"rankEligible":true},{"stateId":"claude-sonnet-5-5::effort:max","configSlug":"claude-sonnet-5-5-max","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5 (max)","modelName":"Claude Sonnet 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":99,"score":167.5,"scoreLo":163.5,"scoreHi":172.37,"reasoningOffset":0.79,"rankEligible":true,"rankMean":5.3219,"rankLo":4.0,"rankHi":7.0},{"stateId":"claude-sonnet-5-5::effort:medium","configSlug":"claude-sonnet-5-5-medium","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5 (medium)","modelName":"Claude Sonnet 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":33,"score":160.75,"scoreLo":157.38,"scoreHi":164.7,"reasoningOffset":-5.96,"rankEligible":true},{"stateId":"claude-sonnet-5-5::effort:xhigh","configSlug":"claude-sonnet-5-5-xhigh","modelId":"claude-sonnet-5-5","modelSlug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5 (xhigh)","modelName":"Claude Sonnet 5.5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":36,"score":166.27,"scoreLo":162.32,"scoreHi":171.0,"reasoningOffset":-0.44,"rankEligible":true},{"stateId":"claude-sonnet-5::effort:adaptive-auto","configSlug":"claude-sonnet-5-adaptive-auto-hle-no-tools-1m","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (adaptive-auto)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"adaptive-auto","reasoningCondition":"effort:adaptive-auto","isDefault":false,"n":2,"score":159.37,"scoreLo":155.14,"scoreHi":164.29,"reasoningOffset":0.48,"rankEligible":false},{"stateId":"claude-sonnet-5::effort:high","configSlug":"claude-sonnet-5-high","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (high)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":53,"score":157.77,"scoreLo":154.76,"scoreHi":161.37,"reasoningOffset":-1.12,"rankEligible":true},{"stateId":"claude-sonnet-5::effort:low","configSlug":"claude-sonnet-5-low","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (low)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":29,"score":153.07,"scoreLo":150.36,"scoreHi":156.22,"reasoningOffset":-5.81,"rankEligible":true},{"stateId":"claude-sonnet-5::effort:max","configSlug":"claude-sonnet-5-max","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (max)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":107,"score":158.98,"scoreLo":155.85,"scoreHi":162.75,"reasoningOffset":0.1,"rankEligible":true,"rankMean":59.9681,"rankLo":53.0,"rankHi":67.0},{"stateId":"claude-sonnet-5::effort:medium","configSlug":"claude-sonnet-5-medium","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (medium)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":34,"score":155.8,"scoreLo":152.89,"scoreHi":159.25,"reasoningOffset":-3.08,"rankEligible":true},{"stateId":"claude-sonnet-5::effort:minimal","configSlug":"claude-sonnet-5-minimal","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (minimal)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":154.73,"scoreLo":149.96,"scoreHi":160.49,"reasoningOffset":-4.16,"rankEligible":false},{"stateId":"claude-sonnet-5::disabled","configSlug":"claude-sonnet-5-no-thinking","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (disabled)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":2,"score":153.03,"scoreLo":147.37,"scoreHi":159.11,"reasoningOffset":-5.86,"rankEligible":false},{"stateId":"claude-sonnet-5::disabled:high","configSlug":"claude-sonnet-5-no-thinking-high","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (disabled, high)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"high","reasoningCondition":"disabled:high","isDefault":false,"n":6,"score":153.3,"scoreLo":149.81,"scoreHi":157.18,"reasoningOffset":-5.58,"rankEligible":true},{"stateId":"claude-sonnet-5::disabled:low","configSlug":"claude-sonnet-5-no-thinking-low-effort-jonathan-chang-visual-acuity","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (disabled, low)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"low","reasoningCondition":"disabled:low","isDefault":false,"n":1,"score":156.16,"scoreLo":150.61,"scoreHi":163.45,"reasoningOffset":-2.73,"rankEligible":false},{"stateId":"claude-sonnet-5::effort:xhigh","configSlug":"claude-sonnet-5-xhigh","modelId":"claude-sonnet-5","modelSlug":"claude-sonnet-5","name":"Sonnet 5 (xhigh)","modelName":"Sonnet 5","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":39,"score":158.45,"scoreLo":155.34,"scoreHi":162.11,"reasoningOffset":-0.43,"rankEligible":true},{"stateId":"codellama-13b-hf::default","configSlug":"codellama-13b-hf","modelId":"codellama-13b-hf","modelSlug":"codellama-13b-hf","name":"Code Llama 13B","modelName":"Code Llama 13B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":87.75,"scoreLo":78.37,"scoreHi":95.7,"reasoningOffset":0.0,"rankEligible":true,"rankMean":801.09,"rankLo":772.0,"rankHi":826.0},{"stateId":"codellama-13b-instruct-hf::default","configSlug":"codellama-13b-instruct-hf","modelId":"codellama-13b-instruct-hf","modelSlug":"codellama-13b-instruct-hf","name":"Code Llama Instruct 13B","modelName":"Code Llama Instruct 13B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":92.15,"scoreLo":83.36,"scoreHi":99.53,"reasoningOffset":0.0,"rankEligible":true,"rankMean":776.5372,"rankLo":739.0,"rankHi":807.0},{"stateId":"codellama-13b-python-hf::default","configSlug":"codellama-13b-python-hf","modelId":"codellama-13b-python-hf","modelSlug":"codellama-13b-python-hf","name":"Code Llama Python 13B","modelName":"Code Llama Python 13B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":87.71,"scoreLo":78.19,"scoreHi":95.81,"reasoningOffset":0.0,"rankEligible":true,"rankMean":801.2153,"rankLo":772.0,"rankHi":826.0},{"stateId":"codellama-34b-hf::default","configSlug":"codellama-34b-hf","modelId":"codellama-34b-hf","modelSlug":"codellama-34b-hf","name":"Code Llama 34B","modelName":"Code Llama 34B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":94.45,"scoreLo":86.08,"scoreHi":101.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":761.2583,"rankLo":722.0,"rankHi":795.0},{"stateId":"codellama-34b-instruct-hf::default","configSlug":"codellama-34b-instruct-hf","modelId":"codellama-34b-instruct-hf","modelSlug":"codellama-34b-instruct-hf","name":"Code Llama Instruct 34B","modelName":"Code Llama Instruct 34B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":92.87,"scoreLo":84.6,"scoreHi":99.72,"reasoningOffset":0.0,"rankEligible":true,"rankMean":772.3292,"rankLo":740.0,"rankHi":799.0},{"stateId":"codellama-34b-python-hf::default","configSlug":"codellama-34b-python-hf","modelId":"codellama-34b-python-hf","modelSlug":"codellama-34b-python-hf","name":"Code Llama Python 34B","modelName":"Code Llama Python 34B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":93.6,"scoreLo":85.45,"scoreHi":100.75,"reasoningOffset":0.0,"rankEligible":true,"rankMean":767.265,"rankLo":729.975,"rankHi":799.0},{"stateId":"codellama-70b-hf::default","configSlug":"codellama-70b-hf","modelId":"codellama-70b-hf","modelSlug":"codellama-70b-hf","name":"Code Llama 70B","modelName":"Code Llama 70B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":102.77,"scoreLo":95.02,"scoreHi":109.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":695.3378,"rankLo":645.0,"rankHi":738.0},{"stateId":"codellama-70b-instruct-hf::default","configSlug":"codellama-70b-instruct","modelId":"codellama-70b-instruct-hf","modelSlug":"codellama-70b-instruct-hf","name":"Code Llama Instruct 70B","modelName":"Code Llama Instruct 70B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":102.89,"scoreLo":95.64,"scoreHi":109.03,"reasoningOffset":0.0,"rankEligible":true,"rankMean":694.4317,"rankLo":651.0,"rankHi":733.0},{"stateId":"codellama-70b-python-hf::default","configSlug":"codellama-70b-python-hf","modelId":"codellama-70b-python-hf","modelSlug":"codellama-70b-python-hf","name":"Code Llama Python 70B","modelName":"Code Llama Python 70B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":105.89,"scoreLo":97.32,"scoreHi":113.54,"reasoningOffset":0.0,"rankEligible":true,"rankMean":665.5497,"rankLo":592.975,"rankHi":725.025},{"stateId":"codellama-7b-hf::default","configSlug":"codellama-7b-hf","modelId":"codellama-7b-hf","modelSlug":"codellama-7b-hf","name":"Code Llama 7B","modelName":"Code Llama 7B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":82.22,"scoreLo":71.46,"scoreHi":91.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":824.9206,"rankLo":802.0,"rankHi":846.0},{"stateId":"codellama-7b-instruct-hf::default","configSlug":"codellama-7b-instruct-hf","modelId":"codellama-7b-instruct-hf","modelSlug":"codellama-7b-instruct-hf","name":"Code Llama Instruct 7B","modelName":"Code Llama Instruct 7B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":86.3,"scoreLo":76.13,"scoreHi":94.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":807.8569,"rankLo":779.0,"rankHi":833.0},{"stateId":"codellama-7b-python-hf::default","configSlug":"codellama-7b-python-hf","modelId":"codellama-7b-python-hf","modelSlug":"codellama-7b-python-hf","name":"Code Llama Python 7B","modelName":"Code Llama Python 7B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":83.31,"scoreLo":73.2,"scoreHi":91.97,"reasoningOffset":0.0,"rankEligible":true,"rankMean":820.7058,"rankLo":798.0,"rankHi":843.025},{"stateId":"codeqwen1-5-7b::default","configSlug":"codeqwen1-5-7b","modelId":"codeqwen1-5-7b","modelSlug":"codeqwen1-5-7b","name":"CodeQwen1.5 7B","modelName":"CodeQwen1.5 7B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":83.71,"scoreLo":74.72,"scoreHi":91.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":819.2711,"rankLo":801.0,"rankHi":837.0},{"stateId":"codeqwen1-5-7b-chat::default","configSlug":"codeqwen1-5-7b-chat","modelId":"codeqwen1-5-7b-chat","modelSlug":"codeqwen1-5-7b-chat","name":"CodeQwen1.5 7B Chat","modelName":"CodeQwen1.5 7B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":86.79,"scoreLo":77.01,"scoreHi":95.34,"reasoningOffset":0.0,"rankEligible":true,"rankMean":805.5894,"rankLo":775.0,"rankHi":831.0},{"stateId":"codestral-22b-v0-1::default","configSlug":"codestral-22b-v0-1","modelId":"codestral-22b-v0-1","modelSlug":"codestral-22b-v0-1","name":"Codestral 2405","modelName":"Codestral 2405","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":108.01,"scoreLo":102.16,"scoreHi":113.03,"reasoningOffset":0.0,"rankEligible":true,"rankMean":645.5219,"rankLo":604.0,"rankHi":684.0},{"stateId":"codestral-2501::default","configSlug":"codestral-2501","modelId":"codestral-2501","modelSlug":"codestral-2501","name":"Codestral 25.01","modelName":"Codestral 25.01","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":120.7,"scoreLo":114.82,"scoreHi":125.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":492.3436,"rankLo":427.0,"rankHi":555.0},{"stateId":"codestral-2508::default","configSlug":"codestral-2508","modelId":"codestral-2508","modelSlug":"codestral-2508","name":"Codestral 25.08","modelName":"Codestral 25.08","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":131.2,"scoreLo":124.28,"scoreHi":137.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":358.3689,"rankLo":272.0,"rankHi":441.0},{"stateId":"cogito-671b-v2-1::enabled","configSlug":"cogito-671b-v2-1-thinking","modelId":"cogito-671b-v2-1","modelSlug":"cogito-671b-v2-1","name":"Cogito v2.1 671B (thinking)","modelName":"Cogito v2.1 671B","organizationSlug":"deepcogito","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":8,"score":137.53,"scoreLo":133.04,"scoreHi":142.03,"reasoningOffset":1.07,"rankEligible":true,"rankMean":273.7861,"rankLo":217.0,"rankHi":334.025},{"stateId":"command-a-plus-05-2026-bf16::default","configSlug":"command-a-plus-05-2026-bf16","modelId":"command-a-plus-05-2026-bf16","modelSlug":"command-a-plus-05-2026-bf16","name":"Command A+ (May 2026, BF16)","modelName":"Command A+ (May 2026, BF16)","organizationSlug":"cohere","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":138.16,"scoreLo":135.34,"scoreHi":141.08,"reasoningOffset":0.0,"rankEligible":true,"rankMean":264.9372,"rankLo":231.0,"rankHi":301.0},{"stateId":"composer-2-5::default","configSlug":"composer-2-5","modelId":"composer-2-5","modelSlug":"composer-2-5","name":"Composer 2.5","modelName":"Composer 2.5","organizationSlug":"cursor","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":155.24,"scoreLo":152.31,"scoreHi":158.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":83.5406,"rankLo":77.0,"rankHi":94.0},{"stateId":"composer-2-5::effort:adaptive","configSlug":"composer-2-5-adaptive-cursor-cli","modelId":"composer-2-5","modelSlug":"composer-2-5","name":"Composer 2.5 (adaptive)","modelName":"Composer 2.5","organizationSlug":"cursor","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"adaptive","reasoningCondition":"effort:adaptive","isDefault":false,"n":1,"score":155.5,"scoreLo":150.09,"scoreHi":162.34,"reasoningOffset":0.26,"rankEligible":false},{"stateId":"dbrx-instruct::default","configSlug":"dbrx-instruct","modelId":"dbrx-instruct","modelSlug":"dbrx-instruct","name":"DBRX Instruct","modelName":"DBRX Instruct","organizationSlug":"databricks","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":109.3,"scoreLo":104.52,"scoreHi":113.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":631.4575,"rankLo":602.0,"rankHi":659.0},{"stateId":"deepseek-v3-1-terminus::default","configSlug":"deepseek-chat-2025-10-28","modelId":"deepseek-v3-1-terminus","modelSlug":"deepseek-v3-1-terminus","name":"DeepSeek-V3.1-Terminus","modelName":"DeepSeek-V3.1-Terminus","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":144.16,"scoreLo":141.69,"scoreHi":146.79,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-coder-1-3b-base::default","configSlug":"deepseek-coder-1-3b-base","modelId":"deepseek-coder-1-3b-base","modelSlug":"deepseek-coder-1-3b-base","name":"DeepSeek-Coder-Base 1.3B","modelName":"DeepSeek-Coder-Base 1.3B","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":57.87,"scoreLo":43.02,"scoreHi":69.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":874.1683,"rankLo":866.0,"rankHi":881.0},{"stateId":"deepseek-coder-1-3b-instruct::default","configSlug":"deepseek-coder-1-3b-instruct","modelId":"deepseek-coder-1-3b-instruct","modelSlug":"deepseek-coder-1-3b-instruct","name":"DeepSeek-Coder-Instruct 1.3B","modelName":"DeepSeek-Coder-Instruct 1.3B","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":67.97,"scoreLo":53.96,"scoreHi":79.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":862.5967,"rankLo":849.0,"rankHi":873.0},{"stateId":"deepseek-coder-33b-base::default","configSlug":"deepseek-coder-33b-base","modelId":"deepseek-coder-33b-base","modelSlug":"deepseek-coder-33b-base","name":"DeepSeek-Coder-Base 33B","modelName":"DeepSeek-Coder-Base 33B","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":82.1,"scoreLo":72.46,"scoreHi":90.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":825.5233,"rankLo":807.0,"rankHi":843.0},{"stateId":"deepseek-coder-33b-instruct::default","configSlug":"deepseek-coder-33b-instruct","modelId":"deepseek-coder-33b-instruct","modelSlug":"deepseek-coder-33b-instruct","name":"DeepSeek-Coder-Instruct 33B","modelName":"DeepSeek-Coder-Instruct 33B","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":111.89,"scoreLo":105.86,"scoreHi":117.86,"reasoningOffset":0.0,"rankEligible":true,"rankMean":600.1989,"rankLo":537.0,"rankHi":656.0},{"stateId":"deepseek-coder-6-7b-base::default","configSlug":"deepseek-coder-6-7b-base","modelId":"deepseek-coder-6-7b-base","modelSlug":"deepseek-coder-6-7b-base","name":"DeepSeek-Coder-Base 6.7B","modelName":"DeepSeek-Coder-Base 6.7B","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":75.66,"scoreLo":65.27,"scoreHi":84.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":846.9619,"rankLo":834.0,"rankHi":859.0},{"stateId":"deepseek-coder-6-7b-instruct::default","configSlug":"deepseek-coder-6-7b-instruct","modelId":"deepseek-coder-6-7b-instruct","modelSlug":"deepseek-coder-6-7b-instruct","name":"DeepSeek-Coder-Instruct 6.7B","modelName":"DeepSeek-Coder-Instruct 6.7B","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":81.16,"scoreLo":71.06,"scoreHi":89.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":829.0072,"rankLo":810.0,"rankHi":847.0},{"stateId":"deepseek-coder-7b-base-v1-5::default","configSlug":"deepseek-coder-7b-base-v1-5","modelId":"deepseek-coder-7b-base-v1-5","modelSlug":"deepseek-coder-7b-base-v1-5","name":"DeepSeek-Coder-Base 7B v1.5","modelName":"DeepSeek-Coder-Base 7B v1.5","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":90.18,"scoreLo":80.45,"scoreHi":98.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":787.9503,"rankLo":750.0,"rankHi":819.0},{"stateId":"deepseek-coder-7b-instruct-v1-5::default","configSlug":"deepseek-coder-7b-instruct-v1-5","modelId":"deepseek-coder-7b-instruct-v1-5","modelSlug":"deepseek-coder-7b-instruct-v1-5","name":"DeepSeek-Coder-Instruct 7B v1.5","modelName":"DeepSeek-Coder-Instruct 7B v1.5","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":89.76,"scoreLo":81.04,"scoreHi":97.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":791.0214,"rankLo":767.0,"rankHi":813.0},{"stateId":"deepseek-coder-v2-instruct::default","configSlug":"deepseek-coder-v2-instruct","modelId":"deepseek-coder-v2-instruct","modelSlug":"deepseek-coder-v2-instruct","name":"DeepSeek-Coder-V2-Instruct","modelName":"DeepSeek-Coder-V2-Instruct","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":122.81,"scoreLo":119.39,"scoreHi":126.01,"reasoningOffset":0.0,"rankEligible":true,"rankMean":464.0358,"rankLo":433.0,"rankHi":499.0},{"stateId":"deepseek-coder-v2-instruct-0724::default","configSlug":"deepseek-coder-v2-instruct-0724","modelId":"deepseek-coder-v2-instruct-0724","modelSlug":"deepseek-coder-v2-instruct-0724","name":"DeepSeek-Coder-V2-Instruct-0724","modelName":"DeepSeek-Coder-V2-Instruct-0724","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":121.95,"scoreLo":118.08,"scoreHi":125.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":476.0989,"rankLo":438.0,"rankHi":518.0},{"stateId":"deepseek-coder-v2-lite-base::default","configSlug":"deepseek-coder-v2-lite-base","modelId":"deepseek-coder-v2-lite-base","modelSlug":"deepseek-coder-v2-lite-base","name":"DeepSeek-Coder-V2-Lite-Base","modelName":"DeepSeek-Coder-V2-Lite-Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":98.92,"scoreLo":91.89,"scoreHi":105.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":727.1172,"rankLo":696.0,"rankHi":761.0},{"stateId":"deepseek-coder-v2-lite-instruct::default","configSlug":"deepseek-coder-v2-lite-instruct","modelId":"deepseek-coder-v2-lite-instruct","modelSlug":"deepseek-coder-v2-lite-instruct","name":"DeepSeek-Coder-V2-Lite-Instruct","modelName":"DeepSeek-Coder-V2-Lite-Instruct","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":107.39,"scoreLo":102.24,"scoreHi":111.96,"reasoningOffset":0.0,"rankEligible":true,"rankMean":652.6294,"rankLo":623.0,"rankHi":680.0},{"stateId":"deepseek-llm-67b-base::default","configSlug":"deepseek-llm-67b-base","modelId":"deepseek-llm-67b-base","modelSlug":"deepseek-llm-67b-base","name":"DeepSeek LLM 67B Base","modelName":"DeepSeek LLM 67B Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":30,"score":107.75,"scoreLo":102.37,"scoreHi":112.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":648.6197,"rankLo":618.0,"rankHi":677.0},{"stateId":"deepseek-llm-67b-chat::default","configSlug":"deepseek-llm-67b-chat","modelId":"deepseek-llm-67b-chat","modelSlug":"deepseek-llm-67b-chat","name":"DeepSeek LLM 67B Chat","modelName":"DeepSeek LLM 67B Chat","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":43,"score":108.42,"scoreLo":103.57,"scoreHi":112.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":641.4928,"rankLo":618.0,"rankHi":664.0},{"stateId":"deepseek-llm-7b-base::default","configSlug":"deepseek-llm-7b-base","modelId":"deepseek-llm-7b-base","modelSlug":"deepseek-llm-7b-base","name":"DeepSeek LLM 7B Base","modelName":"DeepSeek LLM 7B Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":83.92,"scoreLo":74.78,"scoreHi":91.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":818.4383,"rankLo":800.0,"rankHi":836.0},{"stateId":"deepseek-llm-7b-chat::default","configSlug":"deepseek-llm-7b-chat","modelId":"deepseek-llm-7b-chat","modelSlug":"deepseek-llm-7b-chat","name":"DeepSeek LLM 7B Chat","modelName":"DeepSeek LLM 7B Chat","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":90.0,"scoreLo":81.98,"scoreHi":96.71,"reasoningOffset":0.0,"rankEligible":true,"rankMean":790.0019,"rankLo":770.0,"rankHi":808.0},{"stateId":"deepseek-math-7b-base::default","configSlug":"deepseek-math-7b-base","modelId":"deepseek-math-7b-base","modelSlug":"deepseek-math-7b-base","name":"DeepSeekMath 7B Base","modelName":"DeepSeekMath 7B Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":95.88,"scoreLo":88.49,"scoreHi":102.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":751.2736,"rankLo":719.0,"rankHi":780.0},{"stateId":"deepseek-math-7b-instruct::default","configSlug":"deepseek-math-7b-instruct","modelId":"deepseek-math-7b-instruct","modelSlug":"deepseek-math-7b-instruct","name":"DeepSeekMath 7B Instruct","modelName":"DeepSeekMath 7B Instruct","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":96.77,"scoreLo":88.78,"scoreHi":103.73,"reasoningOffset":0.0,"rankEligible":true,"rankMean":744.0892,"rankLo":705.0,"rankHi":780.0},{"stateId":"deepseek-math-7b-rl::default","configSlug":"deepseek-math-7b-rl","modelId":"deepseek-math-7b-rl","modelSlug":"deepseek-math-7b-rl","name":"DeepSeekMath 7B RL","modelName":"DeepSeekMath 7B RL","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":94.95,"scoreLo":87.5,"scoreHi":101.71,"reasoningOffset":0.0,"rankEligible":true,"rankMean":758.1389,"rankLo":725.0,"rankHi":786.0},{"stateId":"deepseek-moe-16b-base::default","configSlug":"deepseek-moe-16b-base","modelId":"deepseek-moe-16b-base","modelSlug":"deepseek-moe-16b-base","name":"DeepSeekMoE 16B Base","modelName":"DeepSeekMoE 16B Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":84.71,"scoreLo":76.1,"scoreHi":92.17,"reasoningOffset":0.0,"rankEligible":true,"rankMean":815.285,"rankLo":799.0,"rankHi":831.0},{"stateId":"deepseek-moe-16b-chat::default","configSlug":"deepseek-moe-16b-chat","modelId":"deepseek-moe-16b-chat","modelSlug":"deepseek-moe-16b-chat","name":"DeepSeekMoE 16B Chat","modelName":"DeepSeekMoE 16B Chat","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":89.99,"scoreLo":81.69,"scoreHi":96.96,"reasoningOffset":0.0,"rankEligible":true,"rankMean":789.8017,"rankLo":765.0,"rankHi":811.025},{"stateId":"deepseek-r1::default","configSlug":"deepseek-r1","modelId":"deepseek-r1","modelSlug":"deepseek-r1","name":"DeepSeek-R1","modelName":"DeepSeek-R1","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":90,"score":137.13,"scoreLo":135.46,"scoreHi":138.85,"reasoningOffset":2.4,"rankEligible":true,"rankMean":279.1297,"rankLo":263.0,"rankHi":294.0},{"stateId":"deepseek-r1-0528::default","configSlug":"deepseek-r1-0528","modelId":"deepseek-r1-0528","modelSlug":"deepseek-r1-0528","name":"DeepSeek-R1-0528","modelName":"DeepSeek-R1-0528","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":76,"score":140.39,"scoreLo":138.69,"scoreHi":142.23,"reasoningOffset":2.4,"rankEligible":true,"rankMean":234.6056,"rankLo":220.0,"rankHi":251.0},{"stateId":"deepseek-r1-0528::effort:minimal","configSlug":"deepseek-r1-0528-minimal","modelId":"deepseek-r1-0528","modelSlug":"deepseek-r1-0528","name":"DeepSeek-R1-0528 (minimal)","modelName":"DeepSeek-R1-0528","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":134.05,"scoreLo":129.37,"scoreHi":138.89,"reasoningOffset":-3.93,"rankEligible":false},{"stateId":"deepseek-r1-0528-qwen3-8b::default","configSlug":"deepseek-r1-0528-qwen3-8b","modelId":"deepseek-r1-0528-qwen3-8b","modelSlug":"deepseek-r1-0528-qwen3-8b","name":"DeepSeek R1 0528 Qwen3 8B","modelName":"DeepSeek R1 0528 Qwen3 8B","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":127.48,"scoreLo":123.61,"scoreHi":130.92,"reasoningOffset":2.4,"rankEligible":true,"rankMean":406.1022,"rankLo":367.0,"rankHi":445.0},{"stateId":"deepseek-r1-distill-llama-70b::default","configSlug":"deepseek-r1-distill-llama-70b","modelId":"deepseek-r1-distill-llama-70b","modelSlug":"deepseek-r1-distill-llama-70b","name":"DeepSeek-R1-Distill-Llama-70B","modelName":"DeepSeek-R1-Distill-Llama-70B","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":30,"score":125.16,"scoreLo":121.9,"scoreHi":128.01,"reasoningOffset":2.4,"rankEligible":true,"rankMean":433.7142,"rankLo":406.0,"rankHi":465.0},{"stateId":"deepseek-r1-distill-llama-8b::default","configSlug":"deepseek-r1-distill-llama-8b","modelId":"deepseek-r1-distill-llama-8b","modelSlug":"deepseek-r1-distill-llama-8b","name":"DeepSeek R1 Distill Llama 8B","modelName":"DeepSeek R1 Distill Llama 8B","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":105.76,"scoreLo":99.84,"scoreHi":110.87,"reasoningOffset":2.4,"rankEligible":true,"rankMean":668.9175,"rankLo":634.0,"rankHi":703.0},{"stateId":"deepseek-r1-distill-qwen-1-5b::default","configSlug":"deepseek-r1-distill-qwen-1-5b","modelId":"deepseek-r1-distill-qwen-1-5b","modelSlug":"deepseek-r1-distill-qwen-1-5b","name":"DeepSeek-R1-Distill-Qwen-1.5B","modelName":"DeepSeek-R1-Distill-Qwen-1.5B","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":86.79,"scoreLo":77.56,"scoreHi":94.5,"reasoningOffset":2.4,"rankEligible":true,"rankMean":805.9383,"rankLo":781.0,"rankHi":827.0},{"stateId":"deepseek-r1-distill-qwen-14b::default","configSlug":"deepseek-r1-distill-qwen-14b","modelId":"deepseek-r1-distill-qwen-14b","modelSlug":"deepseek-r1-distill-qwen-14b","name":"DeepSeek R1 Distill Qwen 14B","modelName":"DeepSeek R1 Distill Qwen 14B","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":31,"score":122.39,"scoreLo":118.94,"scoreHi":125.47,"reasoningOffset":2.4,"rankEligible":true,"rankMean":470.0439,"rankLo":438.0,"rankHi":504.0},{"stateId":"deepseek-r1-distill-qwen-32b::default","configSlug":"deepseek-r1-distill-qwen-32b","modelId":"deepseek-r1-distill-qwen-32b","modelSlug":"deepseek-r1-distill-qwen-32b","name":"DeepSeek-R1-Distill-Qwen-32B","modelName":"DeepSeek-R1-Distill-Qwen-32B","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":33,"score":125.33,"scoreLo":122.37,"scoreHi":128.02,"reasoningOffset":2.4,"rankEligible":true,"rankMean":431.4314,"rankLo":407.0,"rankHi":459.0},{"stateId":"deepseek-r1-distill-qwen-7b::default","configSlug":"deepseek-r1-distill-qwen-7b","modelId":"deepseek-r1-distill-qwen-7b","modelSlug":"deepseek-r1-distill-qwen-7b","name":"DeepSeek-R1-Distill-Qwen-7B","modelName":"DeepSeek-R1-Distill-Qwen-7B","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":111.71,"scoreLo":101.58,"scoreHi":119.61,"reasoningOffset":2.4,"rankEligible":true,"rankMean":601.0331,"rankLo":514.0,"rankHi":701.0},{"stateId":"deepseek-r1-zero::default","configSlug":"deepseek-r1-zero","modelId":"deepseek-r1-zero","modelSlug":"deepseek-r1-zero","name":"DeepSeek-R1-Zero","modelName":"DeepSeek-R1-Zero","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":130.13,"scoreLo":126.54,"scoreHi":133.53,"reasoningOffset":2.4,"rankEligible":true,"rankMean":373.0128,"rankLo":332.0,"rankHi":412.0},{"stateId":"deepseek-v2::default","configSlug":"deepseek-v2","modelId":"deepseek-v2","modelSlug":"deepseek-v2","name":"DeepSeek-V2","modelName":"DeepSeek-V2","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":115.08,"scoreLo":110.72,"scoreHi":118.94,"reasoningOffset":0.0,"rankEligible":true,"rankMean":560.9303,"rankLo":529.0,"rankHi":593.0},{"stateId":"deepseek-v2-5::default","configSlug":"deepseek-v2-5","modelId":"deepseek-v2-5","modelSlug":"deepseek-v2-5","name":"DeepSeek-V2.5","modelName":"DeepSeek-V2.5","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":35,"score":122.44,"scoreLo":119.32,"scoreHi":125.12,"reasoningOffset":0.0,"rankEligible":true,"rankMean":469.1906,"rankLo":443.0,"rankHi":497.0},{"stateId":"deepseek-v2-chat::default","configSlug":"deepseek-v2-chat","modelId":"deepseek-v2-chat","modelSlug":"deepseek-v2-chat","name":"DeepSeek-V2-Chat","modelName":"DeepSeek-V2-Chat","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":33,"score":117.83,"scoreLo":114.07,"scoreHi":121.16,"reasoningOffset":0.0,"rankEligible":true,"rankMean":529.1456,"rankLo":504.0,"rankHi":553.0},{"stateId":"deepseek-v2-chat-0628::default","configSlug":"deepseek-v2-chat-0628","modelId":"deepseek-v2-chat-0628","modelSlug":"deepseek-v2-chat-0628","name":"DeepSeek-V2-Chat-0628","modelName":"DeepSeek-V2-Chat-0628","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":120.71,"scoreLo":117.09,"scoreHi":124.02,"reasoningOffset":0.0,"rankEligible":true,"rankMean":493.3475,"rankLo":457.975,"rankHi":526.0},{"stateId":"deepseek-v2-lite::default","configSlug":"deepseek-v2-lite","modelId":"deepseek-v2-lite","modelSlug":"deepseek-v2-lite","name":"DeepSeek-V2-Lite","modelName":"DeepSeek-V2-Lite","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":91.02,"scoreLo":82.52,"scoreHi":98.36,"reasoningOffset":0.0,"rankEligible":true,"rankMean":783.8689,"rankLo":757.0,"rankHi":808.0},{"stateId":"deepseek-v2-lite-chat::default","configSlug":"deepseek-v2-lite-chat","modelId":"deepseek-v2-lite-chat","modelSlug":"deepseek-v2-lite-chat","name":"DeepSeek-V2-Lite-Chat","modelName":"DeepSeek-V2-Lite-Chat","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":97.74,"scoreLo":90.6,"scoreHi":103.63,"reasoningOffset":0.0,"rankEligible":true,"rankMean":736.4536,"rankLo":711.0,"rankHi":763.0},{"stateId":"deepseek-v3::default","configSlug":"deepseek-v3","modelId":"deepseek-v3","modelSlug":"deepseek-v3","name":"DeepSeek-V3","modelName":"DeepSeek-V3","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":84,"score":131.56,"scoreLo":129.54,"scoreHi":133.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":353.8836,"rankLo":335.0,"rankHi":374.0},{"stateId":"deepseek-v3-0324::default","configSlug":"deepseek-v3-0324","modelId":"deepseek-v3-0324","modelSlug":"deepseek-v3-0324","name":"DeepSeek-V3-0324","modelName":"DeepSeek-V3-0324","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":67,"score":135.06,"scoreLo":133.23,"scoreHi":136.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":307.1128,"rankLo":287.0,"rankHi":328.0},{"stateId":"deepseek-v3-1::default","configSlug":"deepseek-v3-1","modelId":"deepseek-v3-1","modelSlug":"deepseek-v3-1","name":"DeepSeek-V3.1","modelName":"DeepSeek-V3.1","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":23,"score":142.36,"scoreLo":140.32,"scoreHi":144.65,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v3-1::effort:high","configSlug":"deepseek-v3-1-high","modelId":"deepseek-v3-1","modelSlug":"deepseek-v3-1","name":"DeepSeek-V3.1 (high)","modelName":"DeepSeek-V3.1","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":142.64,"scoreLo":139.5,"scoreHi":146.01,"reasoningOffset":0.27,"rankEligible":false},{"stateId":"deepseek-v3-1::disabled","configSlug":"deepseek-v3-1-no-thinking","modelId":"deepseek-v3-1","modelSlug":"deepseek-v3-1","name":"DeepSeek-V3.1 (disabled)","modelName":"DeepSeek-V3.1","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":22,"score":140.61,"scoreLo":138.62,"scoreHi":142.72,"reasoningOffset":-1.75,"rankEligible":true},{"stateId":"deepseek-v3-1-terminus::disabled","configSlug":"deepseek-v3-1-terminus-no-thinking","modelId":"deepseek-v3-1-terminus","modelSlug":"deepseek-v3-1-terminus","name":"DeepSeek-V3.1-Terminus (disabled)","modelName":"DeepSeek-V3.1-Terminus","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":140.5,"scoreLo":137.23,"scoreHi":143.84,"reasoningOffset":-3.66,"rankEligible":true},{"stateId":"deepseek-v3-1-terminus::enabled","configSlug":"deepseek-v3-1-terminus-thinking","modelId":"deepseek-v3-1-terminus","modelSlug":"deepseek-v3-1-terminus","name":"DeepSeek-V3.1-Terminus (thinking)","modelName":"DeepSeek-V3.1-Terminus","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":24,"score":145.39,"scoreLo":143.11,"scoreHi":147.89,"reasoningOffset":1.22,"rankEligible":true,"rankMean":181.9353,"rankLo":165.0,"rankHi":200.0},{"stateId":"deepseek-v3-1::enabled","configSlug":"deepseek-v3-1-thinking","modelId":"deepseek-v3-1","modelSlug":"deepseek-v3-1","name":"DeepSeek-V3.1 (thinking)","modelName":"DeepSeek-V3.1","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":29,"score":143.69,"scoreLo":141.81,"scoreHi":145.76,"reasoningOffset":1.33,"rankEligible":true,"rankMean":200.0289,"rankLo":187.0,"rankHi":211.0},{"stateId":"deepseek-v3-2::default","configSlug":"deepseek-v3-2","modelId":"deepseek-v3-2","modelSlug":"deepseek-v3-2","name":"DeepSeek-V3.2","modelName":"DeepSeek-V3.2","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":55,"score":146.27,"scoreLo":144.2,"scoreHi":148.49,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v3-2-exp::default","configSlug":"deepseek-v3-2-exp","modelId":"deepseek-v3-2-exp","modelSlug":"deepseek-v3-2-exp","name":"DeepSeek-V3.2-Exp","modelName":"DeepSeek-V3.2-Exp","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":15,"score":144.45,"scoreLo":142.13,"scoreHi":147.05,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v3-2-exp::effort:high","configSlug":"deepseek-v3-2-exp-high","modelId":"deepseek-v3-2-exp","modelSlug":"deepseek-v3-2-exp","name":"DeepSeek-V3.2-Exp (high)","modelName":"DeepSeek-V3.2-Exp","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":144.87,"scoreLo":141.57,"scoreHi":148.49,"reasoningOffset":0.42,"rankEligible":false},{"stateId":"deepseek-v3-2-exp::disabled","configSlug":"deepseek-v3-2-exp-no-thinking","modelId":"deepseek-v3-2-exp","modelSlug":"deepseek-v3-2-exp","name":"DeepSeek-V3.2-Exp (disabled)","modelName":"DeepSeek-V3.2-Exp","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":11,"score":141.82,"scoreLo":138.84,"scoreHi":144.9,"reasoningOffset":-2.63,"rankEligible":true},{"stateId":"deepseek-v3-2-exp::enabled","configSlug":"deepseek-v3-2-exp-thinking","modelId":"deepseek-v3-2-exp","modelSlug":"deepseek-v3-2-exp","name":"DeepSeek-V3.2-Exp (thinking)","modelName":"DeepSeek-V3.2-Exp","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":23,"score":145.41,"scoreLo":143.26,"scoreHi":147.8,"reasoningOffset":0.96,"rankEligible":true,"rankMean":181.7181,"rankLo":167.0,"rankHi":197.0},{"stateId":"deepseek-v3-2::effort:high","configSlug":"deepseek-v3-2-high","modelId":"deepseek-v3-2","modelSlug":"deepseek-v3-2","name":"DeepSeek-V3.2 (high)","modelName":"DeepSeek-V3.2","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":14,"score":146.34,"scoreLo":143.85,"scoreHi":149.04,"reasoningOffset":0.08,"rankEligible":true},{"stateId":"deepseek-v3-2::effort:minimal","configSlug":"deepseek-v3-2-minimal","modelId":"deepseek-v3-2","modelSlug":"deepseek-v3-2","name":"DeepSeek-V3.2 (minimal)","modelName":"DeepSeek-V3.2","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":142.1,"scoreLo":137.59,"scoreHi":146.94,"reasoningOffset":-4.17,"rankEligible":false},{"stateId":"deepseek-v3-2::disabled","configSlug":"deepseek-v3-2-no-thinking","modelId":"deepseek-v3-2","modelSlug":"deepseek-v3-2","name":"DeepSeek-V3.2 (disabled)","modelName":"DeepSeek-V3.2","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":24,"score":143.57,"scoreLo":141.23,"scoreHi":146.06,"reasoningOffset":-2.7,"rankEligible":true},{"stateId":"deepseek-v3-2-speciale::default","configSlug":"deepseek-v3-2-speciale-thinking","modelId":"deepseek-v3-2-speciale","modelSlug":"deepseek-v3-2-speciale","name":"DeepSeek-V3.2-Speciale","modelName":"DeepSeek-V3.2-Speciale","organizationSlug":"deepseek","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":23,"score":149.22,"scoreLo":146.31,"scoreHi":152.52,"reasoningOffset":2.4,"rankEligible":true,"rankMean":140.2375,"rankLo":118.0,"rankHi":163.0},{"stateId":"deepseek-v3-2::enabled","configSlug":"deepseek-v3-2-thinking","modelId":"deepseek-v3-2","modelSlug":"deepseek-v3-2","name":"DeepSeek-V3.2 (thinking)","modelName":"DeepSeek-V3.2","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":52,"score":147.74,"scoreLo":145.69,"scoreHi":150.09,"reasoningOffset":1.48,"rankEligible":true,"rankMean":156.6236,"rankLo":146.0,"rankHi":167.0},{"stateId":"deepseek-v3-base::default","configSlug":"deepseek-v3-base","modelId":"deepseek-v3-base","modelSlug":"deepseek-v3-base","name":"DeepSeek-V3-Base","modelName":"DeepSeek-V3-Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":30,"score":123.94,"scoreLo":120.41,"scoreHi":127.3,"reasoningOffset":0.0,"rankEligible":true,"rankMean":449.2222,"rankLo":415.0,"rankHi":484.0},{"stateId":"deepseek-v4-1-flash::default","configSlug":"deepseek-v4-1-flash","modelId":"deepseek-v4-1-flash","modelSlug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","modelName":"DeepSeek V4.1 Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":49,"score":161.25,"scoreLo":157.91,"scoreHi":165.3,"reasoningOffset":0.0,"rankEligible":true,"rankMean":35.3808,"rankLo":28.0,"rankHi":43.025},{"stateId":"deepseek-v4-1-flash-1017::default","configSlug":"deepseek-v4-1-flash-1017","modelId":"deepseek-v4-1-flash-1017","modelSlug":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017","modelName":"DeepSeek V4.1 Flash 1017","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":162.61,"scoreLo":159.08,"scoreHi":166.85,"reasoningOffset":0.0,"rankEligible":true,"rankMean":23.315,"rankLo":19.0,"rankHi":31.0},{"stateId":"deepseek-v4-1-flash-1017::enabled","configSlug":"deepseek-v4-1-flash-1017-concept-synth-cascade","modelId":"deepseek-v4-1-flash-1017","modelSlug":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017 (enabled)","modelName":"DeepSeek V4.1 Flash 1017","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":162.81,"scoreLo":158.7,"scoreHi":167.67,"reasoningOffset":0.2,"rankEligible":false},{"stateId":"deepseek-v4-1-flash-1017::effort:high","configSlug":"deepseek-v4-1-flash-1017-high","modelId":"deepseek-v4-1-flash-1017","modelSlug":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017 (high)","modelName":"DeepSeek V4.1 Flash 1017","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":15,"score":161.79,"scoreLo":158.33,"scoreHi":165.95,"reasoningOffset":-0.82,"rankEligible":true},{"stateId":"deepseek-v4-1-flash-1017::effort:low","configSlug":"deepseek-v4-1-flash-1017-low","modelId":"deepseek-v4-1-flash-1017","modelSlug":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017 (low)","modelName":"DeepSeek V4.1 Flash 1017","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":5,"score":162.08,"scoreLo":158.59,"scoreHi":166.37,"reasoningOffset":-0.53,"rankEligible":true},{"stateId":"deepseek-v4-1-flash-1017::effort:max","configSlug":"deepseek-v4-1-flash-1017-max","modelId":"deepseek-v4-1-flash-1017","modelSlug":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017 (max)","modelName":"DeepSeek V4.1 Flash 1017","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":64,"score":161.98,"scoreLo":158.55,"scoreHi":166.12,"reasoningOffset":-0.63,"rankEligible":true},{"stateId":"deepseek-v4-1-flash-1017::disabled","configSlug":"deepseek-v4-1-flash-1017-no-thinking","modelId":"deepseek-v4-1-flash-1017","modelSlug":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017 (disabled)","modelName":"DeepSeek V4.1 Flash 1017","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":null,"reasoningCondition":"disabled","isDefault":false,"n":13,"score":150.49,"scoreLo":147.79,"scoreHi":153.6,"reasoningOffset":-12.12,"rankEligible":true},{"stateId":"deepseek-v4-1-flash::effort:high","configSlug":"deepseek-v4-1-flash-high","modelId":"deepseek-v4-1-flash","modelSlug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash (high)","modelName":"DeepSeek V4.1 Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":38,"score":160.3,"scoreLo":156.99,"scoreHi":164.2,"reasoningOffset":-0.95,"rankEligible":true},{"stateId":"deepseek-v4-1-flash::effort:low","configSlug":"deepseek-v4-1-flash-low","modelId":"deepseek-v4-1-flash","modelSlug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash (low)","modelName":"DeepSeek V4.1 Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":6,"score":160.14,"scoreLo":156.76,"scoreHi":164.13,"reasoningOffset":-1.12,"rankEligible":true},{"stateId":"deepseek-v4-1-flash::effort:max","configSlug":"deepseek-v4-1-flash-max","modelId":"deepseek-v4-1-flash","modelSlug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash (max)","modelName":"DeepSeek V4.1 Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":101,"score":160.57,"scoreLo":157.32,"scoreHi":164.46,"reasoningOffset":-0.69,"rankEligible":true},{"stateId":"deepseek-v4-1-flash::disabled","configSlug":"deepseek-v4-1-flash-no-thinking","modelId":"deepseek-v4-1-flash","modelSlug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash (disabled)","modelName":"DeepSeek V4.1 Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":null,"reasoningCondition":"disabled","isDefault":false,"n":15,"score":149.63,"scoreLo":146.67,"scoreHi":152.81,"reasoningOffset":-11.63,"rankEligible":true},{"stateId":"deepseek-v4-flash::default","configSlug":"deepseek-v4-flash","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":38,"score":151.98,"scoreLo":149.51,"scoreHi":154.91,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v4-flash-0731::default","configSlug":"deepseek-v4-flash-0731","modelId":"deepseek-v4-flash-0731","modelSlug":"deepseek-v4-flash-0731","name":"DeepSeek-V4-Flash-0731","modelName":"DeepSeek-V4-Flash-0731","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":157.02,"scoreLo":154.06,"scoreHi":160.61,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v4-flash-0731::effort:high","configSlug":"deepseek-v4-flash-0731-high","modelId":"deepseek-v4-flash-0731","modelSlug":"deepseek-v4-flash-0731","name":"DeepSeek-V4-Flash-0731 (high)","modelName":"DeepSeek-V4-Flash-0731","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":33,"score":157.27,"scoreLo":154.22,"scoreHi":160.81,"reasoningOffset":0.25,"rankEligible":true},{"stateId":"deepseek-v4-flash-0731::effort:low","configSlug":"deepseek-v4-flash-0731-low","modelId":"deepseek-v4-flash-0731","modelSlug":"deepseek-v4-flash-0731","name":"DeepSeek-V4-Flash-0731 (low)","modelName":"DeepSeek-V4-Flash-0731","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":3,"score":156.58,"scoreLo":153.53,"scoreHi":160.29,"reasoningOffset":-0.45,"rankEligible":false},{"stateId":"deepseek-v4-flash-0731::effort:max","configSlug":"deepseek-v4-flash-0731-max","modelId":"deepseek-v4-flash-0731","modelSlug":"deepseek-v4-flash-0731","name":"DeepSeek-V4-Flash-0731 (max)","modelName":"DeepSeek-V4-Flash-0731","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":23,"score":157.79,"scoreLo":154.73,"scoreHi":161.48,"reasoningOffset":0.76,"rankEligible":true,"rankMean":69.7669,"rankLo":62.0,"rankHi":76.0},{"stateId":"deepseek-v4-flash-0731::effort:medium","configSlug":"deepseek-v4-flash-0731-medium-pi","modelId":"deepseek-v4-flash-0731","modelSlug":"deepseek-v4-flash-0731","name":"DeepSeek-V4-Flash-0731 (medium)","modelName":"DeepSeek-V4-Flash-0731","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":157.02,"scoreLo":152.82,"scoreHi":161.64,"reasoningOffset":-0.01,"rankEligible":false},{"stateId":"deepseek-v4-flash-0731::disabled","configSlug":"deepseek-v4-flash-0731-no-thinking","modelId":"deepseek-v4-flash-0731","modelSlug":"deepseek-v4-flash-0731","name":"DeepSeek-V4-Flash-0731 (disabled)","modelName":"DeepSeek-V4-Flash-0731","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":null,"reasoningCondition":"disabled","isDefault":false,"n":5,"score":148.75,"scoreLo":145.87,"scoreHi":151.76,"reasoningOffset":-8.28,"rankEligible":true},{"stateId":"deepseek-v4-flash-base::default","configSlug":"deepseek-v4-flash-base","modelId":"deepseek-v4-flash-base","modelSlug":"deepseek-v4-flash-base","name":"DeepSeek-V4-Flash-Base","modelName":"DeepSeek-V4-Flash-Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":19,"score":129.24,"scoreLo":126.19,"scoreHi":132.34,"reasoningOffset":0.0,"rankEligible":true,"rankMean":384.8044,"rankLo":348.0,"rankHi":416.0},{"stateId":"deepseek-v4-flash::effort:high","configSlug":"deepseek-v4-flash-high","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (high)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":39,"score":151.97,"scoreLo":149.47,"scoreHi":154.88,"reasoningOffset":-0.01,"rankEligible":true},{"stateId":"deepseek-v4-flash::effort:low","configSlug":"deepseek-v4-flash-low","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (low)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":149.16,"scoreLo":144.59,"scoreHi":154.14,"reasoningOffset":-2.83,"rankEligible":false},{"stateId":"deepseek-v4-flash::effort:max","configSlug":"deepseek-v4-flash-max","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (max)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":37,"score":153.41,"scoreLo":150.73,"scoreHi":156.48,"reasoningOffset":1.43,"rankEligible":true,"rankMean":98.2308,"rankLo":88.0,"rankHi":108.0},{"stateId":"deepseek-v4-flash::effort:medium","configSlug":"deepseek-v4-flash-medium","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (medium)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":151.63,"scoreLo":147.43,"scoreHi":156.25,"reasoningOffset":-0.35,"rankEligible":false},{"stateId":"deepseek-v4-flash::effort:minimal","configSlug":"deepseek-v4-flash-minimal","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (minimal)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":147.59,"scoreLo":143.1,"scoreHi":152.47,"reasoningOffset":-4.39,"rankEligible":false},{"stateId":"deepseek-v4-flash::disabled","configSlug":"deepseek-v4-flash-no-thinking","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (disabled)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":23,"score":144.06,"scoreLo":141.85,"scoreHi":146.4,"reasoningOffset":-7.93,"rankEligible":true},{"stateId":"deepseek-v4-flash::enabled","configSlug":"deepseek-v4-flash-thinking","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (thinking)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":2,"score":153.02,"scoreLo":149.07,"scoreHi":157.47,"reasoningOffset":1.04,"rankEligible":false},{"stateId":"deepseek-v4-flash-vision-exp::default","configSlug":"deepseek-v4-flash-vision-exp","modelId":"deepseek-v4-flash-vision-exp","modelSlug":"deepseek-v4-flash-vision-exp","name":"DeepSeek V4 Flash Vision Exp","modelName":"DeepSeek V4 Flash Vision Exp","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":158.21,"scoreLo":154.78,"scoreHi":162.24,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v4-flash-vision-exp::effort:high","configSlug":"deepseek-v4-flash-vision-exp-high-riemann-bench-unconstrained-agent","modelId":"deepseek-v4-flash-vision-exp","modelSlug":"deepseek-v4-flash-vision-exp","name":"DeepSeek V4 Flash Vision Exp (high)","modelName":"DeepSeek V4 Flash Vision Exp","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":158.23,"scoreLo":154.66,"scoreHi":162.4,"reasoningOffset":0.02,"rankEligible":false},{"stateId":"deepseek-v4-flash-vision-exp::effort:max","configSlug":"deepseek-v4-flash-vision-exp-max","modelId":"deepseek-v4-flash-vision-exp","modelSlug":"deepseek-v4-flash-vision-exp","name":"DeepSeek V4 Flash Vision Exp (max)","modelName":"DeepSeek V4 Flash Vision Exp","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":21,"score":158.52,"scoreLo":155.28,"scoreHi":162.4,"reasoningOffset":0.31,"rankEligible":true,"rankMean":63.9603,"rankLo":51.0,"rankHi":73.0},{"stateId":"deepseek-v4-flash::effort:xhigh","configSlug":"deepseek-v4-flash-xhigh","modelId":"deepseek-v4-flash","modelSlug":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (xhigh)","modelName":"DeepSeek-V4-Flash","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":3,"score":153.0,"scoreLo":149.86,"scoreHi":156.77,"reasoningOffset":1.02,"rankEligible":false},{"stateId":"deepseek-v4-pro::default","configSlug":"deepseek-v4-pro","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":57,"score":153.18,"scoreLo":150.6,"scoreHi":156.21,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v4-pro-0813::default","configSlug":"deepseek-v4-pro-0813","modelId":"deepseek-v4-pro-0813","modelSlug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","modelName":"DeepSeek V4 Pro 0813","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":159.15,"scoreLo":155.98,"scoreHi":163.02,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"deepseek-v4-pro-0813::effort:high","configSlug":"deepseek-v4-pro-0813-high","modelId":"deepseek-v4-pro-0813","modelSlug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813 (high)","modelName":"DeepSeek V4 Pro 0813","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":11,"score":158.89,"scoreLo":155.68,"scoreHi":162.77,"reasoningOffset":-0.26,"rankEligible":true},{"stateId":"deepseek-v4-pro-0813::effort:low","configSlug":"deepseek-v4-pro-0813-low","modelId":"deepseek-v4-pro-0813","modelSlug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813 (low)","modelName":"DeepSeek V4 Pro 0813","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":4,"score":158.75,"scoreLo":155.49,"scoreHi":162.65,"reasoningOffset":-0.39,"rankEligible":true},{"stateId":"deepseek-v4-pro-0813::effort:max","configSlug":"deepseek-v4-pro-0813-max","modelId":"deepseek-v4-pro-0813","modelSlug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813 (max)","modelName":"DeepSeek V4 Pro 0813","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":77,"score":159.45,"scoreLo":156.3,"scoreHi":163.31,"reasoningOffset":0.3,"rankEligible":true,"rankMean":54.7067,"rankLo":47.0,"rankHi":63.0},{"stateId":"deepseek-v4-pro-0813::disabled","configSlug":"deepseek-v4-pro-0813-no-thinking","modelId":"deepseek-v4-pro-0813","modelSlug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813 (disabled)","modelName":"DeepSeek V4 Pro 0813","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":null,"reasoningCondition":"disabled","isDefault":false,"n":9,"score":149.43,"scoreLo":146.59,"scoreHi":152.54,"reasoningOffset":-9.71,"rankEligible":true},{"stateId":"deepseek-v4-pro-base::default","configSlug":"deepseek-v4-pro-base","modelId":"deepseek-v4-pro-base","modelSlug":"deepseek-v4-pro-base","name":"DeepSeek-V4-Pro-Base","modelName":"DeepSeek-V4-Pro-Base","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":19,"score":131.97,"scoreLo":128.97,"scoreHi":135.0,"reasoningOffset":0.0,"rankEligible":true,"rankMean":348.8444,"rankLo":312.0,"rankHi":384.0},{"stateId":"deepseek-v4-pro::effort:high","configSlug":"deepseek-v4-pro-high","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (high)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":39,"score":153.61,"scoreLo":150.94,"scoreHi":156.78,"reasoningOffset":0.43,"rankEligible":true},{"stateId":"deepseek-v4-pro::effort:low","configSlug":"deepseek-v4-pro-low","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (low)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":151.01,"scoreLo":146.54,"scoreHi":155.88,"reasoningOffset":-2.17,"rankEligible":false},{"stateId":"deepseek-v4-pro::effort:max","configSlug":"deepseek-v4-pro-max","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (max)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":89,"score":154.08,"scoreLo":151.43,"scoreHi":157.1,"reasoningOffset":0.89,"rankEligible":true,"rankMean":92.0628,"rankLo":85.0,"rankHi":100.0},{"stateId":"deepseek-v4-pro::effort:medium","configSlug":"deepseek-v4-pro-medium","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (medium)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":152.84,"scoreLo":148.62,"scoreHi":157.51,"reasoningOffset":-0.35,"rankEligible":false},{"stateId":"deepseek-v4-pro::effort:minimal","configSlug":"deepseek-v4-pro-minimal","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (minimal)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":149.05,"scoreLo":144.37,"scoreHi":154.38,"reasoningOffset":-4.13,"rankEligible":false},{"stateId":"deepseek-v4-pro::disabled","configSlug":"deepseek-v4-pro-no-thinking","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (disabled)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":28,"score":144.54,"scoreLo":142.34,"scoreHi":146.86,"reasoningOffset":-8.65,"rankEligible":true},{"stateId":"deepseek-v4-pro::enabled","configSlug":"deepseek-v4-pro-thinking-context-arena-mrcr-v2","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (thinking)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":154.37,"scoreLo":150.48,"scoreHi":158.75,"reasoningOffset":1.2,"rankEligible":false},{"stateId":"deepseek-v4-pro::effort:xhigh","configSlug":"deepseek-v4-pro-xhigh","modelId":"deepseek-v4-pro","modelSlug":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (xhigh)","modelName":"DeepSeek-V4-Pro","organizationSlug":"deepseek","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":5,"score":153.04,"scoreLo":150.26,"scoreHi":156.35,"reasoningOffset":-0.14,"rankEligible":true},{"stateId":"deepseek-vl-1-3b-chat::default","configSlug":"deepseek-vl-1-3b-chat","modelId":"deepseek-vl-1-3b-chat","modelSlug":"deepseek-vl-1-3b-chat","name":"DeepSeek-VL 1.3B Chat","modelName":"DeepSeek-VL 1.3B Chat","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":72.81,"scoreLo":60.77,"scoreHi":82.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":853.9636,"rankLo":838.0,"rankHi":866.0},{"stateId":"deepseek-vl-7b-chat::default","configSlug":"deepseek-vl-7b","modelId":"deepseek-vl-7b-chat","modelSlug":"deepseek-vl-7b-chat","name":"DeepSeek-VL 7B Chat","modelName":"DeepSeek-VL 7B Chat","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":92.06,"scoreLo":83.95,"scoreHi":99.0,"reasoningOffset":0.0,"rankEligible":true,"rankMean":777.5625,"rankLo":750.975,"rankHi":802.0},{"stateId":"deepseek-vl2::default","configSlug":"deepseek-vl2","modelId":"deepseek-vl2","modelSlug":"deepseek-vl2","name":"DeepSeek-VL2","modelName":"DeepSeek-VL2","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":113.5,"scoreLo":107.09,"scoreHi":118.87,"reasoningOffset":0.0,"rankEligible":true,"rankMean":580.5475,"rankLo":524.0,"rankHi":642.0},{"stateId":"deepseek-vl2-small::default","configSlug":"deepseek-vl2-small","modelId":"deepseek-vl2-small","modelSlug":"deepseek-vl2-small","name":"DeepSeek-VL2-Small","modelName":"DeepSeek-VL2-Small","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":111.25,"scoreLo":104.57,"scoreHi":117.06,"reasoningOffset":0.0,"rankEligible":true,"rankMean":607.6644,"rankLo":549.0,"rankHi":666.0},{"stateId":"deepseek-vl2-tiny::default","configSlug":"deepseek-vl2-tiny","modelId":"deepseek-vl2-tiny","modelSlug":"deepseek-vl2-tiny","name":"DeepSeek-VL2-Tiny","modelName":"DeepSeek-VL2-Tiny","organizationSlug":"deepseek","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":102.58,"scoreLo":95.06,"scoreHi":109.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":697.0636,"rankLo":649.0,"rankHi":742.0},{"stateId":"devstral-2-123b-instruct-2512::default","configSlug":"devstral-2-123b-instruct-2512","modelId":"devstral-2-123b-instruct-2512","modelSlug":"devstral-2-123b-instruct-2512","name":"Devstral 2","modelName":"Devstral 2","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":139.47,"scoreLo":136.99,"scoreHi":141.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":246.9981,"rankLo":220.0,"rankHi":277.0},{"stateId":"devstral-medium-2507::default","configSlug":"devstral-medium-2507","modelId":"devstral-medium-2507","modelSlug":"devstral-medium-2507","name":"Devstral Medium 1.0","modelName":"Devstral Medium 1.0","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":130.68,"scoreLo":126.82,"scoreHi":134.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":365.7164,"rankLo":320.0,"rankHi":410.0},{"stateId":"devstral-small-2-24b-instruct-2512::default","configSlug":"devstral-small-2-24b-instruct-2512","modelId":"devstral-small-2-24b-instruct-2512","modelSlug":"devstral-small-2-24b-instruct-2512","name":"Devstral Small 2","modelName":"Devstral Small 2","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":135.49,"scoreLo":132.56,"scoreHi":138.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":301.4217,"rankLo":268.0,"rankHi":338.0},{"stateId":"devstral-small-2505::default","configSlug":"devstral-small-2505","modelId":"devstral-small-2505","modelSlug":"devstral-small-2505","name":"Devstral Small 1.0","modelName":"Devstral Small 1.0","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":127.05,"scoreLo":121.73,"scoreHi":131.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":411.2081,"rankLo":351.0,"rankHi":472.025},{"stateId":"devstral-small-2507::default","configSlug":"devstral-small-2507","modelId":"devstral-small-2507","modelSlug":"devstral-small-2507","name":"Devstral Small 1.1","modelName":"Devstral Small 1.1","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":123.64,"scoreLo":118.43,"scoreHi":128.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":453.6719,"rankLo":403.0,"rankHi":515.0},{"stateId":"diffusiongemma-26b-a4b-it::disabled","configSlug":"diffusiongemma-26b-a4b-it-no-thinking","modelId":"diffusiongemma-26b-a4b-it","modelSlug":"diffusiongemma-26b-a4b-it","name":"DiffusionGemma 26B A4B IT (disabled)","modelName":"DiffusionGemma 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":14,"score":134.35,"scoreLo":131.85,"scoreHi":136.78,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"diffusiongemma-26b-a4b-it::enabled","configSlug":"diffusiongemma-26b-a4b-it-thinking","modelId":"diffusiongemma-26b-a4b-it","modelSlug":"diffusiongemma-26b-a4b-it","name":"DiffusionGemma 26B A4B IT (thinking)","modelName":"DiffusionGemma 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":24,"score":138.19,"scoreLo":136.16,"scoreHi":140.19,"reasoningOffset":-0.16,"rankEligible":true,"rankMean":264.6442,"rankLo":244.0,"rankHi":286.0},{"stateId":"dolly-v2-12b::default","configSlug":"dolly-v2-12b","modelId":"dolly-v2-12b","modelSlug":"dolly-v2-12b","name":"Dolly 2.0 12B","modelName":"Dolly 2.0 12B","organizationSlug":"databricks","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":67.13,"scoreLo":54.55,"scoreHi":77.25,"reasoningOffset":0.0,"rankEligible":true,"rankMean":864.1594,"rankLo":855.0,"rankHi":871.0},{"stateId":"doubao-seed-2-0-pro-260215::default","configSlug":"doubao-seed-2-0-pro-260215","modelId":"doubao-seed-2-0-pro-260215","modelSlug":"doubao-seed-2-0-pro-260215","name":"Doubao Seed 2.0 Pro","modelName":"Doubao Seed 2.0 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":151.05,"scoreLo":148.03,"scoreHi":154.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":120.7019,"rankLo":99.0,"rankHi":146.0},{"stateId":"doubao-seed-2-0-pro-260215::effort:high","configSlug":"doubao-seed-2-0-pro-260215-high","modelId":"doubao-seed-2-0-pro-260215","modelSlug":"doubao-seed-2-0-pro-260215","name":"Doubao Seed 2.0 Pro (high)","modelName":"Doubao Seed 2.0 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":152.31,"scoreLo":148.39,"scoreHi":156.4,"reasoningOffset":1.26,"rankEligible":false},{"stateId":"doubao-seed-2-0-pro-260215::effort:low","configSlug":"doubao-seed-2-0-pro-260215-low","modelId":"doubao-seed-2-0-pro-260215","modelSlug":"doubao-seed-2-0-pro-260215","name":"Doubao Seed 2.0 Pro (low)","modelName":"Doubao Seed 2.0 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":148.94,"scoreLo":144.47,"scoreHi":153.52,"reasoningOffset":-2.1,"rankEligible":false},{"stateId":"doubao-seed-2-0-pro-260215::effort:medium","configSlug":"doubao-seed-2-0-pro-260215-medium","modelId":"doubao-seed-2-0-pro-260215","modelSlug":"doubao-seed-2-0-pro-260215","name":"Doubao Seed 2.0 Pro (medium)","modelName":"Doubao Seed 2.0 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":150.24,"scoreLo":146.12,"scoreHi":154.96,"reasoningOffset":-0.8,"rankEligible":false},{"stateId":"doubao-seed-2-0-pro-260215::disabled","configSlug":"doubao-seed-2-0-pro-260215-no-thinking","modelId":"doubao-seed-2-0-pro-260215","modelSlug":"doubao-seed-2-0-pro-260215","name":"Doubao Seed 2.0 Pro (disabled)","modelName":"Doubao Seed 2.0 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":144.91,"scoreLo":138.76,"scoreHi":151.02,"reasoningOffset":-6.14,"rankEligible":false},{"stateId":"doubao-seed-2-1-pro-260628::default","configSlug":"doubao-seed-2-1-pro-260628","modelId":"doubao-seed-2-1-pro-260628","modelSlug":"doubao-seed-2-1-pro-260628","name":"Doubao Seed 2.1 Pro","modelName":"Doubao Seed 2.1 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":155.84,"scoreLo":151.81,"scoreHi":160.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":80.8808,"rankLo":63.0,"rankHi":104.0},{"stateId":"doubao-seed-2-1-pro-260628::effort:high","configSlug":"doubao-seed-2-1-pro-260628-high","modelId":"doubao-seed-2-1-pro-260628","modelSlug":"doubao-seed-2-1-pro-260628","name":"Doubao Seed 2.1 Pro (high)","modelName":"Doubao Seed 2.1 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":156.06,"scoreLo":151.69,"scoreHi":160.88,"reasoningOffset":0.22,"rankEligible":false},{"stateId":"doubao-seed-2-1-pro-260628::enabled","configSlug":"doubao-seed-2-1-pro-260628-thinking-terminus-2","modelId":"doubao-seed-2-1-pro-260628","modelSlug":"doubao-seed-2-1-pro-260628","name":"Doubao Seed 2.1 Pro (thinking)","modelName":"Doubao Seed 2.1 Pro","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":156.6,"scoreLo":151.75,"scoreHi":161.98,"reasoningOffset":0.76,"rankEligible":false},{"stateId":"doubao-seed-code-preview-251028::enabled","configSlug":"doubao-seed-code-preview-251028-thinking","modelId":"doubao-seed-code-preview-251028","modelSlug":"doubao-seed-code-preview-251028","name":"Doubao-Seed-Code (thinking)","modelName":"Doubao-Seed-Code","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":144.1,"scoreLo":140.51,"scoreHi":147.82,"reasoningOffset":1.07,"rankEligible":true,"rankMean":195.2178,"rankLo":162.0,"rankHi":228.025},{"stateId":"ember-1::default","configSlug":"ember-1","modelId":"ember-1","modelSlug":"ember-1","name":"Ember-1","modelName":"Ember-1","organizationSlug":"fireworks","reasoningSupport":"unknown","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":160.62,"scoreLo":157.26,"scoreHi":164.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":42.2803,"rankLo":30.0,"rankHi":54.0},{"stateId":"ernie-4-5-300b-a47b-pt::default","configSlug":"ernie-4-5-300b-a47b-pt","modelId":"ernie-4-5-300b-a47b-pt","modelSlug":"ernie-4-5-300b-a47b-pt","name":"ERNIE 4.5 300B A47B","modelName":"ERNIE 4.5 300B A47B","organizationSlug":"baidu","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":129.73,"scoreLo":124.54,"scoreHi":134.83,"reasoningOffset":0.0,"rankEligible":true,"rankMean":377.8242,"rankLo":315.0,"rankHi":438.0},{"stateId":"ernie-5-0::default","configSlug":"ernie-5-0","modelId":"ernie-5-0","modelSlug":"ernie-5-0","name":"ERNIE 5.0","modelName":"ERNIE 5.0","organizationSlug":"baidu","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":5,"score":138.87,"scoreLo":134.4,"scoreHi":142.96,"reasoningOffset":2.4,"rankEligible":true,"rankMean":255.7675,"rankLo":210.0,"rankHi":313.0},{"stateId":"ernie-5-0-thinking-preview::default","configSlug":"ernie-5-0-thinking-preview","modelId":"ernie-5-0-thinking-preview","modelSlug":"ernie-5-0-thinking-preview","name":"ERNIE 5.0 Thinking Preview","modelName":"ERNIE 5.0 Thinking Preview","organizationSlug":"baidu","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":8,"score":144.44,"scoreLo":140.63,"scoreHi":148.5,"reasoningOffset":0.0,"rankEligible":true,"rankMean":191.6656,"rankLo":155.0,"rankHi":228.0},{"stateId":"ernie-5-1::default","configSlug":"ernie-5-1","modelId":"ernie-5-1","modelSlug":"ernie-5-1","name":"ERNIE 5.1","modelName":"ERNIE 5.1","organizationSlug":"baidu","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":145.4,"scoreLo":141.25,"scoreHi":149.75,"reasoningOffset":0.0,"rankEligible":true,"rankMean":181.5572,"rankLo":140.0,"rankHi":219.0},{"stateId":"exaone-4-0-1-2b::disabled","configSlug":"exaone-4-0-1-2b-no-thinking","modelId":"exaone-4-0-1-2b","modelSlug":"exaone-4-0-1-2b","name":"EXAONE 4.0 1.2B (disabled)","modelName":"EXAONE 4.0 1.2B","organizationSlug":"lg-ai-research","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":114.64,"scoreLo":107.93,"scoreHi":120.63,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"exaone-4-0-1-2b::enabled","configSlug":"exaone-4-0-1-2b-thinking","modelId":"exaone-4-0-1-2b","modelSlug":"exaone-4-0-1-2b","name":"EXAONE 4.0 1.2B (thinking)","modelName":"EXAONE 4.0 1.2B","organizationSlug":"lg-ai-research","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":120.26,"scoreLo":114.45,"scoreHi":125.35,"reasoningOffset":1.63,"rankEligible":true,"rankMean":498.1214,"rankLo":435.0,"rankHi":557.0},{"stateId":"exaone-4-0-32b::disabled","configSlug":"exaone-4-0-32b-no-thinking","modelId":"exaone-4-0-32b","modelSlug":"exaone-4-0-32b","name":"EXAONE 4.0 32B (disabled)","modelName":"EXAONE 4.0 32B","organizationSlug":"lg-ai-research","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":128.26,"scoreLo":124.34,"scoreHi":131.98,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"exaone-4-0-32b::enabled","configSlug":"exaone-4-0-32b-thinking","modelId":"exaone-4-0-32b","modelSlug":"exaone-4-0-32b","name":"EXAONE 4.0 32B (thinking)","modelName":"EXAONE 4.0 32B","organizationSlug":"lg-ai-research","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":134.18,"scoreLo":130.63,"scoreHi":137.61,"reasoningOffset":1.93,"rankEligible":true,"rankMean":319.3467,"rankLo":275.975,"rankHi":363.0},{"stateId":"exaone-4-5-33b::enabled","configSlug":"exaone-4-5-33b-thinking","modelId":"exaone-4-5-33b","modelSlug":"exaone-4-5-33b","name":"EXAONE 4.5 33B (thinking)","modelName":"EXAONE 4.5 33B","organizationSlug":"lg-ai-research","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":141.43,"scoreLo":138.15,"scoreHi":144.95,"reasoningOffset":1.07,"rankEligible":true,"rankMean":223.3822,"rankLo":193.0,"rankHi":263.0},{"stateId":"falcon-11b::default","configSlug":"falcon-11b","modelId":"falcon-11b","modelSlug":"falcon-11b","name":"Falcon2-11B","modelName":"Falcon2-11B","organizationSlug":"tii","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":94.24,"scoreLo":86.23,"scoreHi":100.84,"reasoningOffset":0.0,"rankEligible":true,"rankMean":763.3314,"rankLo":731.975,"rankHi":790.0},{"stateId":"falcon-40b::default","configSlug":"falcon-40b","modelId":"falcon-40b","modelSlug":"falcon-40b","name":"Falcon 40B","modelName":"Falcon 40B","organizationSlug":"tii","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":90.13,"scoreLo":82.57,"scoreHi":96.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":789.38,"rankLo":772.0,"rankHi":806.0},{"stateId":"falcon-40b-instruct::default","configSlug":"falcon-40b-instruct","modelId":"falcon-40b-instruct","modelSlug":"falcon-40b-instruct","name":"Falcon 40B Instruct","modelName":"Falcon 40B Instruct","organizationSlug":"tii","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":91.02,"scoreLo":82.43,"scoreHi":98.07,"reasoningOffset":0.0,"rankEligible":true,"rankMean":783.7489,"rankLo":756.0,"rankHi":808.025},{"stateId":"falcon-7b::default","configSlug":"falcon-7b","modelId":"falcon-7b","modelSlug":"falcon-7b","name":"Falcon 7B","modelName":"Falcon 7B","organizationSlug":"tii","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":37,"score":61.22,"scoreLo":48.59,"scoreHi":71.36,"reasoningOffset":0.0,"rankEligible":true,"rankMean":871.0675,"rankLo":866.0,"rankHi":877.0},{"stateId":"falcon-h1r-7b::default","configSlug":"falcon-h1r-7b","modelId":"falcon-h1r-7b","modelSlug":"falcon-h1r-7b","name":"Falcon H1R 7B","modelName":"Falcon H1R 7B","organizationSlug":"tii","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":135.35,"scoreLo":132.41,"scoreHi":138.13,"reasoningOffset":2.4,"rankEligible":true,"rankMean":303.3283,"rankLo":270.0,"rankHi":339.0},{"stateId":"flan-t5-xxl::default","configSlug":"flan-t5-xxl","modelId":"flan-t5-xxl","modelSlug":"flan-t5-xxl","name":"Flan-T5 XXL","modelName":"Flan-T5 XXL","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":84.95,"scoreLo":75.19,"scoreHi":93.19,"reasoningOffset":0.0,"rankEligible":true,"rankMean":814.0433,"rankLo":790.0,"rankHi":836.0},{"stateId":"g9v3-39a5b::enabled","configSlug":"g9v3-39a5b-thinking","modelId":"g9v3-39a5b","modelSlug":"g9v3-39a5b","name":"G9v3-39A5B (thinking)","modelName":"G9v3-39A5B","organizationSlug":"ai9stars","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":143.82,"scoreLo":139.99,"scoreHi":147.84,"reasoningOffset":1.07,"rankEligible":true,"rankMean":198.2647,"rankLo":162.0,"rankHi":239.0},{"stateId":"g9v3-3b::default","configSlug":"g9v3-3b","modelId":"g9v3-3b","modelSlug":"g9v3-3b","name":"G9v3-3B","modelName":"G9v3-3B","organizationSlug":"ai9stars","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":5,"score":133.67,"scoreLo":127.98,"scoreHi":138.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":325.7986,"rankLo":261.975,"rankHi":399.0},{"stateId":"gemini-1-5-flash-001::default","configSlug":"gemini-1-5-flash-001","modelId":"gemini-1-5-flash-001","modelSlug":"gemini-1-5-flash-001","name":"Gemini 1.5 Flash 001","modelName":"Gemini 1.5 Flash 001","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":116.29,"scoreLo":109.54,"scoreHi":122.16,"reasoningOffset":0.0,"rankEligible":true,"rankMean":547.0975,"rankLo":482.0,"rankHi":616.0},{"stateId":"gemini-1-5-flash-002::default","configSlug":"gemini-1-5-flash-002","modelId":"gemini-1-5-flash-002","modelSlug":"gemini-1-5-flash-002","name":"Gemini 1.5 Flash 002","modelName":"Gemini 1.5 Flash 002","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":42,"score":123.39,"scoreLo":120.24,"scoreHi":126.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":456.2678,"rankLo":429.0,"rankHi":486.025},{"stateId":"gemini-1-5-flash-8b::default","configSlug":"gemini-1-5-flash-8b","modelId":"gemini-1-5-flash-8b","modelSlug":"gemini-1-5-flash-8b","name":"Gemini 1.5 Flash-8B","modelName":"Gemini 1.5 Flash-8B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":114.27,"scoreLo":108.96,"scoreHi":118.97,"reasoningOffset":0.0,"rankEligible":true,"rankMean":571.0269,"rankLo":527.0,"rankHi":619.025},{"stateId":"gemini-1-5-flash-preview-0514::default","configSlug":"gemini-1-5-flash-api-0514-benchmark-entry","modelId":"gemini-1-5-flash-preview-0514","modelSlug":"gemini-1-5-flash-preview-0514","name":"Gemini 1.5 Flash Preview 0514","modelName":"Gemini 1.5 Flash Preview 0514","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":119.68,"scoreLo":114.76,"scoreHi":124.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":506.2206,"rankLo":448.0,"rankHi":553.0},{"stateId":"gemini-1-5-pro-001::default","configSlug":"gemini-1-5-pro-001","modelId":"gemini-1-5-pro-001","modelSlug":"gemini-1-5-pro-001","name":"Gemini 1.5 Pro 001","modelName":"Gemini 1.5 Pro 001","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":122.72,"scoreLo":118.02,"scoreHi":126.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":465.7703,"rankLo":417.0,"rankHi":519.0},{"stateId":"gemini-1-5-pro-002::default","configSlug":"gemini-1-5-pro-002","modelId":"gemini-1-5-pro-002","modelSlug":"gemini-1-5-pro-002","name":"Gemini 1.5 Pro 002","modelName":"Gemini 1.5 Pro 002","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":55,"score":129.42,"scoreLo":126.91,"scoreHi":131.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":382.7183,"rankLo":357.0,"rankHi":406.0},{"stateId":"gemini-1-5-pro-preview-0514::default","configSlug":"gemini-1-5-pro-api-0514-benchmark-entry","modelId":"gemini-1-5-pro-preview-0514","modelSlug":"gemini-1-5-pro-preview-0514","name":"Gemini 1.5 Pro Preview 0514","modelName":"Gemini 1.5 Pro Preview 0514","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":121.88,"scoreLo":117.43,"scoreHi":126.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":477.2342,"rankLo":424.0,"rankHi":526.0},{"stateId":"gemini-1-5-pro-exp-0827::default","configSlug":"gemini-1-5-pro-exp-0827","modelId":"gemini-1-5-pro-exp-0827","modelSlug":"gemini-1-5-pro-exp-0827","name":"Gemini 1.5 Pro Experimental 0827","modelName":"Gemini 1.5 Pro Experimental 0827","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":129.13,"scoreLo":125.48,"scoreHi":132.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":385.9561,"rankLo":341.0,"rankHi":424.025},{"stateId":"gemini-1-5-pro-preview-0409::default","configSlug":"gemini-1-5-pro-preview-0409","modelId":"gemini-1-5-pro-preview-0409","modelSlug":"gemini-1-5-pro-preview-0409","name":"Gemini 1.5 Pro Preview 0409","modelName":"Gemini 1.5 Pro Preview 0409","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":119.61,"scoreLo":114.63,"scoreHi":124.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":506.8861,"rankLo":444.975,"rankHi":554.0},{"stateId":"gemini-2-0-flash::default","configSlug":"gemini-2-0-flash","modelId":"gemini-2-0-flash","modelSlug":"gemini-2-0-flash","name":"Gemini 2.0 Flash","modelName":"Gemini 2.0 Flash","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":73,"score":130.94,"scoreLo":128.9,"scoreHi":132.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":362.0628,"rankLo":343.0,"rankHi":384.0},{"stateId":"gemini-2-0-flash-exp::default","configSlug":"gemini-2-0-flash-exp","modelId":"gemini-2-0-flash-exp","modelSlug":"gemini-2-0-flash-exp","name":"Gemini 2.0 Flash Experimental","modelName":"Gemini 2.0 Flash Experimental","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":32,"score":131.21,"scoreLo":128.75,"scoreHi":133.63,"reasoningOffset":0.0,"rankEligible":true,"rankMean":358.6047,"rankLo":332.0,"rankHi":387.0},{"stateId":"gemini-2-0-flash-lite::default","configSlug":"gemini-2-0-flash-lite","modelId":"gemini-2-0-flash-lite","modelSlug":"gemini-2-0-flash-lite","name":"Gemini 2.0 Flash-Lite","modelName":"Gemini 2.0 Flash-Lite","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":124.45,"scoreLo":121.0,"scoreHi":127.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":442.7033,"rankLo":411.0,"rankHi":478.025},{"stateId":"gemini-2-0-flash-lite-preview-02-05::default","configSlug":"gemini-2-0-flash-lite-preview-02-05","modelId":"gemini-2-0-flash-lite-preview-02-05","modelSlug":"gemini-2-0-flash-lite-preview-02-05","name":"Gemini 2.0 Flash-Lite Preview 02-05","modelName":"Gemini 2.0 Flash-Lite Preview 02-05","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":127.61,"scoreLo":124.16,"scoreHi":130.74,"reasoningOffset":0.0,"rankEligible":true,"rankMean":404.5811,"rankLo":370.0,"rankHi":438.025},{"stateId":"gemini-2-0-flash-thinking-exp-01-21::default","configSlug":"gemini-2-0-flash-thinking-exp","modelId":"gemini-2-0-flash-thinking-exp-01-21","modelSlug":"gemini-2-0-flash-thinking-exp-01-21","name":"Gemini 2.0 Flash Thinking Experimental 01-21","modelName":"Gemini 2.0 Flash Thinking Experimental 01-21","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":20,"score":133.14,"scoreLo":130.23,"scoreHi":136.1,"reasoningOffset":2.4,"rankEligible":true,"rankMean":333.4172,"rankLo":297.0,"rankHi":368.0},{"stateId":"gemini-2-0-pro-exp-02-05::default","configSlug":"gemini-2-0-pro-exp-feb-2025","modelId":"gemini-2-0-pro-exp-02-05","modelSlug":"gemini-2-0-pro-exp-02-05","name":"Gemini 2.0 Pro Experimental 02-05","modelName":"Gemini 2.0 Pro Experimental 02-05","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":133.48,"scoreLo":130.45,"scoreHi":136.43,"reasoningOffset":0.0,"rankEligible":true,"rankMean":328.9008,"rankLo":291.0,"rankHi":364.0},{"stateId":"gemini-2-5-deep-think::default","configSlug":"gemini-2-5-deep-think","modelId":"gemini-2-5-deep-think","modelSlug":"gemini-2-5-deep-think","name":"Gemini 2.5 Deep Think","modelName":"Gemini 2.5 Deep Think","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":4,"score":154.37,"scoreLo":150.28,"scoreHi":159.01,"reasoningOffset":2.4,"rankEligible":true,"rankMean":91.3286,"rankLo":70.975,"rankHi":120.0},{"stateId":"gemini-2-5-flash::default","configSlug":"gemini-2-5-flash","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":67,"score":141.31,"scoreLo":139.62,"scoreHi":143.15,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-2-5-flash::effort:16k","configSlug":"gemini-2-5-flash-16k","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (16k)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":3,"score":141.82,"scoreLo":139.86,"scoreHi":143.94,"reasoningOffset":0.51,"rankEligible":false},{"stateId":"gemini-2-5-flash::effort:high","configSlug":"gemini-2-5-flash-high","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (high)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":8,"score":142.05,"scoreLo":139.95,"scoreHi":144.17,"reasoningOffset":0.74,"rankEligible":true,"rankMean":215.5917,"rankLo":203.0,"rankHi":232.0},{"stateId":"gemini-2-5-flash-lite::default","configSlug":"gemini-2-5-flash-lite","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":45,"score":131.55,"scoreLo":129.42,"scoreHi":133.54,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-2-5-flash-lite::effort:16k","configSlug":"gemini-2-5-flash-lite-16k","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (16k)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":1,"score":131.85,"scoreLo":129.25,"scoreHi":134.35,"reasoningOffset":0.3,"rankEligible":false},{"stateId":"gemini-2-5-flash-lite-preview-09-2025::enabled","configSlug":"gemini-2-5-flash-lite-2025-09-thinking","modelId":"gemini-2-5-flash-lite-preview-09-2025","modelSlug":"gemini-2-5-flash-lite-preview-09-2025","name":"Gemini 2.5 Flash-Lite Preview (09-2025) (thinking)","modelName":"Gemini 2.5 Flash-Lite Preview (09-2025)","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":28,"score":137.52,"scoreLo":135.36,"scoreHi":139.78,"reasoningOffset":0.5,"rankEligible":true,"rankMean":273.6408,"rankLo":249.0,"rankHi":298.0},{"stateId":"gemini-2-5-flash-lite::effort:24576","configSlug":"gemini-2-5-flash-lite-24576-ale-bench-self-refine-x1","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (24576)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"24576","reasoningCondition":"effort:24576","isDefault":false,"n":1,"score":132.94,"scoreLo":128.2,"scoreHi":137.74,"reasoningOffset":1.39,"rankEligible":false},{"stateId":"gemini-2-5-flash-lite::effort:dynamic","configSlug":"gemini-2-5-flash-lite-dynamic","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (dynamic)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"dynamic","reasoningCondition":"effort:dynamic","isDefault":false,"n":7,"score":132.4,"scoreLo":129.56,"scoreHi":135.2,"reasoningOffset":0.86,"rankEligible":true},{"stateId":"gemini-2-5-flash-lite::effort:high","configSlug":"gemini-2-5-flash-lite-high","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (high)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":8,"score":133.38,"scoreLo":130.94,"scoreHi":135.68,"reasoningOffset":1.83,"rankEligible":true,"rankMean":330.5278,"rankLo":304.0,"rankHi":356.0},{"stateId":"gemini-2-5-flash-lite::effort:low","configSlug":"gemini-2-5-flash-lite-low","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (low)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":131.19,"scoreLo":127.4,"scoreHi":134.89,"reasoningOffset":-0.37,"rankEligible":false},{"stateId":"gemini-2-5-flash-lite::effort:medium","configSlug":"gemini-2-5-flash-lite-medium","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (medium)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":132.07,"scoreLo":128.53,"scoreHi":135.75,"reasoningOffset":0.53,"rankEligible":false},{"stateId":"gemini-2-5-flash-lite::disabled","configSlug":"gemini-2-5-flash-lite-no-thinking","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (disabled)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":22,"score":130.65,"scoreLo":128.41,"scoreHi":132.87,"reasoningOffset":-0.89,"rankEligible":true},{"stateId":"gemini-2-5-flash-lite-preview-09-2025::default","configSlug":"gemini-2-5-flash-lite-preview-09-2025","modelId":"gemini-2-5-flash-lite-preview-09-2025","modelSlug":"gemini-2-5-flash-lite-preview-09-2025","name":"Gemini 2.5 Flash-Lite Preview (09-2025)","modelName":"Gemini 2.5 Flash-Lite Preview (09-2025)","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":3,"score":137.03,"scoreLo":134.2,"scoreHi":139.92,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gemini-2-5-flash-lite-preview-09-2025::disabled","configSlug":"gemini-2-5-flash-lite-preview-09-2025-no-thinking","modelId":"gemini-2-5-flash-lite-preview-09-2025","modelSlug":"gemini-2-5-flash-lite-preview-09-2025","name":"Gemini 2.5 Flash-Lite Preview (09-2025) (disabled)","modelName":"Gemini 2.5 Flash-Lite Preview (09-2025)","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":30,"score":136.03,"scoreLo":133.84,"scoreHi":138.21,"reasoningOffset":-1.0,"rankEligible":true},{"stateId":"gemini-2-5-flash-lite::enabled","configSlug":"gemini-2-5-flash-lite-thinking","modelId":"gemini-2-5-flash-lite","modelSlug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite (thinking)","modelName":"Gemini 2.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":23,"score":133.35,"scoreLo":131.43,"scoreHi":135.24,"reasoningOffset":1.81,"rankEligible":true},{"stateId":"gemini-2-5-flash::effort:low","configSlug":"gemini-2-5-flash-low","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (low)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":140.3,"scoreLo":136.84,"scoreHi":144.02,"reasoningOffset":-1.01,"rankEligible":false},{"stateId":"gemini-2-5-flash::effort:medium","configSlug":"gemini-2-5-flash-medium","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (medium)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":141.72,"scoreLo":138.41,"scoreHi":145.32,"reasoningOffset":0.4,"rankEligible":false},{"stateId":"gemini-2-5-flash::disabled","configSlug":"gemini-2-5-flash-no-thinking","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (disabled)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":60,"score":139.68,"scoreLo":138.09,"scoreHi":141.41,"reasoningOffset":-1.63,"rankEligible":true},{"stateId":"gemini-2-5-flash-preview-04-17::default","configSlug":"gemini-2-5-flash-preview-04-17","modelId":"gemini-2-5-flash-preview-04-17","modelSlug":"gemini-2-5-flash-preview-04-17","name":"Gemini 2.5 Flash Preview 04-17","modelName":"Gemini 2.5 Flash Preview 04-17","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":14,"score":136.0,"scoreLo":133.2,"scoreHi":138.78,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-2-5-flash-preview-04-17::disabled","configSlug":"gemini-2-5-flash-preview-04-17-no-thinking","modelId":"gemini-2-5-flash-preview-04-17","modelSlug":"gemini-2-5-flash-preview-04-17","name":"Gemini 2.5 Flash Preview 04-17 (disabled)","modelName":"Gemini 2.5 Flash Preview 04-17","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":13,"score":134.39,"scoreLo":131.9,"scoreHi":136.84,"reasoningOffset":-1.61,"rankEligible":true},{"stateId":"gemini-2-5-flash-preview-04-17::enabled","configSlug":"gemini-2-5-flash-preview-04-17-thinking","modelId":"gemini-2-5-flash-preview-04-17","modelSlug":"gemini-2-5-flash-preview-04-17","name":"Gemini 2.5 Flash Preview 04-17 (thinking)","modelName":"Gemini 2.5 Flash Preview 04-17","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":20,"score":137.47,"scoreLo":135.13,"scoreHi":139.78,"reasoningOffset":1.47,"rankEligible":true,"rankMean":274.4103,"rankLo":248.0,"rankHi":300.0},{"stateId":"gemini-2-5-flash::effort:1k","configSlug":"gemini-2-5-flash-preview-05-20-1k","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (1k)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":2,"score":139.87,"scoreLo":137.7,"scoreHi":142.04,"reasoningOffset":-1.44,"rankEligible":false},{"stateId":"gemini-2-5-flash::effort:24576","configSlug":"gemini-2-5-flash-preview-05-20-24576","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (24576)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"24576","reasoningCondition":"effort:24576","isDefault":false,"n":2,"score":142.62,"scoreLo":138.03,"scoreHi":147.33,"reasoningOffset":1.31,"rankEligible":false},{"stateId":"gemini-2-5-flash::effort:24k","configSlug":"gemini-2-5-flash-preview-05-20-24k","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (24k)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"24k","reasoningCondition":"effort:24k","isDefault":false,"n":3,"score":142.09,"scoreLo":139.89,"scoreHi":144.4,"reasoningOffset":0.77,"rankEligible":false},{"stateId":"gemini-2-5-flash::effort:8k","configSlug":"gemini-2-5-flash-preview-05-20-8k","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (8k)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":2,"score":141.16,"scoreLo":139.17,"scoreHi":143.23,"reasoningOffset":-0.16,"rankEligible":false},{"stateId":"gemini-2-5-flash-preview-09-2025::default","configSlug":"gemini-2-5-flash-preview-09-2025","modelId":"gemini-2-5-flash-preview-09-2025","modelSlug":"gemini-2-5-flash-preview-09-2025","name":"Gemini 2.5 Flash Preview (09-2025)","modelName":"Gemini 2.5 Flash Preview (09-2025)","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":142.47,"scoreLo":139.57,"scoreHi":145.51,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-2-5-flash-preview-09-2025::effort:16k","configSlug":"gemini-2-5-flash-preview-09-2025-16k","modelId":"gemini-2-5-flash-preview-09-2025","modelSlug":"gemini-2-5-flash-preview-09-2025","name":"Gemini 2.5 Flash Preview (09-2025) (16k)","modelName":"Gemini 2.5 Flash Preview (09-2025)","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":1,"score":142.72,"scoreLo":139.5,"scoreHi":146.14,"reasoningOffset":0.24,"rankEligible":false},{"stateId":"gemini-2-5-flash-preview-09-2025::disabled","configSlug":"gemini-2-5-flash-preview-09-2025-no-thinking","modelId":"gemini-2-5-flash-preview-09-2025","modelSlug":"gemini-2-5-flash-preview-09-2025","name":"Gemini 2.5 Flash Preview (09-2025) (disabled)","modelName":"Gemini 2.5 Flash Preview (09-2025)","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":20,"score":143.13,"scoreLo":140.68,"scoreHi":145.85,"reasoningOffset":0.66,"rankEligible":true},{"stateId":"gemini-2-5-flash-preview-09-2025::enabled","configSlug":"gemini-2-5-flash-sep-25-thinking","modelId":"gemini-2-5-flash-preview-09-2025","modelSlug":"gemini-2-5-flash-preview-09-2025","name":"Gemini 2.5 Flash Preview (09-2025) (thinking)","modelName":"Gemini 2.5 Flash Preview (09-2025)","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":31,"score":143.43,"scoreLo":141.17,"scoreHi":145.96,"reasoningOffset":0.96,"rankEligible":true,"rankMean":202.3253,"rankLo":185.0,"rankHi":219.0},{"stateId":"gemini-2-5-flash::enabled","configSlug":"gemini-2-5-flash-thinking","modelId":"gemini-2-5-flash","modelSlug":"gemini-2-5-flash","name":"Gemini 2.5 Flash (thinking)","modelName":"Gemini 2.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":72,"score":141.23,"scoreLo":139.59,"scoreHi":143.03,"reasoningOffset":-0.09,"rankEligible":true},{"stateId":"gemini-2-5-pro::default","configSlug":"gemini-2-5-pro","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":116,"score":144.9,"scoreLo":143.07,"scoreHi":146.91,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-2-5-pro::effort:16k","configSlug":"gemini-2-5-pro-16k","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (16k)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":3,"score":145.14,"scoreLo":143.15,"scoreHi":147.38,"reasoningOffset":0.25,"rankEligible":false},{"stateId":"gemini-2-5-pro::effort:1k","configSlug":"gemini-2-5-pro-1k","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (1k)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":2,"score":142.61,"scoreLo":140.36,"scoreHi":144.87,"reasoningOffset":-2.29,"rankEligible":false},{"stateId":"gemini-2-5-pro::effort:32k","configSlug":"gemini-2-5-pro-32k","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (32k)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"32k","reasoningCondition":"effort:32k","isDefault":false,"n":4,"score":144.84,"scoreLo":142.83,"scoreHi":147.14,"reasoningOffset":-0.05,"rankEligible":true},{"stateId":"gemini-2-5-pro::effort:8k","configSlug":"gemini-2-5-pro-8k","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (8k)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"8k","reasoningCondition":"effort:8k","isDefault":false,"n":2,"score":144.32,"scoreLo":142.3,"scoreHi":146.52,"reasoningOffset":-0.57,"rankEligible":false},{"stateId":"gemini-2-5-pro-exp-03-25::default","configSlug":"gemini-2-5-pro-exp-03-25","modelId":"gemini-2-5-pro-exp-03-25","modelSlug":"gemini-2-5-pro-exp-03-25","name":"Gemini 2.5 Pro Experimental 03-25","modelName":"Gemini 2.5 Pro Experimental 03-25","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":146.06,"scoreLo":143.2,"scoreHi":149.28,"reasoningOffset":2.4,"rankEligible":true,"rankMean":174.7042,"rankLo":148.0,"rankHi":199.0},{"stateId":"gemini-2-5-pro-preview-03-25::effort:high","configSlug":"gemini-2-5-pro-exp-high","modelId":"gemini-2-5-pro-preview-03-25","modelSlug":"gemini-2-5-pro-preview-03-25","name":"Gemini 2.5 Pro Preview 03-25 (high)","modelName":"Gemini 2.5 Pro Preview 03-25","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":142.74,"scoreLo":139.34,"scoreHi":146.11,"reasoningOffset":1.01,"rankEligible":false},{"stateId":"gemini-2-5-pro::effort:high","configSlug":"gemini-2-5-pro-high","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (high)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":18,"score":146.22,"scoreLo":144.14,"scoreHi":148.54,"reasoningOffset":1.32,"rankEligible":true,"rankMean":173.0119,"rankLo":161.0,"rankHi":185.0},{"stateId":"gemini-2-5-pro::effort:low","configSlug":"gemini-2-5-pro-low","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (low)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":143.96,"scoreLo":140.63,"scoreHi":147.66,"reasoningOffset":-0.94,"rankEligible":false},{"stateId":"gemini-2-5-pro-preview-05-06::default","configSlug":"gemini-2-5-pro-may-25","modelId":"gemini-2-5-pro-preview-05-06","modelSlug":"gemini-2-5-pro-preview-05-06","name":"Gemini 2.5 Pro Preview 05-06","modelName":"Gemini 2.5 Pro Preview 05-06","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":143.49,"scoreLo":140.71,"scoreHi":146.55,"reasoningOffset":2.4,"rankEligible":true,"rankMean":201.5911,"rankLo":176.0,"rankHi":225.0},{"stateId":"gemini-2-5-pro::effort:medium","configSlug":"gemini-2-5-pro-medium","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (medium)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":145.31,"scoreLo":141.99,"scoreHi":149.03,"reasoningOffset":0.41,"rankEligible":false},{"stateId":"gemini-2-5-pro::effort:minimal","configSlug":"gemini-2-5-pro-minimal","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (minimal)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":140.83,"scoreLo":136.49,"scoreHi":145.8,"reasoningOffset":-4.08,"rankEligible":false},{"stateId":"gemini-2-5-pro-preview-03-25::default","configSlug":"gemini-2-5-pro-preview-03-25","modelId":"gemini-2-5-pro-preview-03-25","modelSlug":"gemini-2-5-pro-preview-03-25","name":"Gemini 2.5 Pro Preview 03-25","modelName":"Gemini 2.5 Pro Preview 03-25","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":144.13,"scoreLo":141.73,"scoreHi":146.64,"reasoningOffset":2.4,"rankEligible":true,"rankMean":195.1753,"rankLo":175.0,"rankHi":214.0},{"stateId":"gemini-2-5-pro-preview-05-06::effort:128","configSlug":"gemini-2-5-pro-preview-05-06-128","modelId":"gemini-2-5-pro-preview-05-06","modelSlug":"gemini-2-5-pro-preview-05-06","name":"Gemini 2.5 Pro Preview 05-06 (128)","modelName":"Gemini 2.5 Pro Preview 05-06","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"128","reasoningCondition":"effort:128","isDefault":false,"n":1,"score":142.66,"scoreLo":137.16,"scoreHi":148.57,"reasoningOffset":1.57,"rankEligible":false},{"stateId":"gemini-2-5-pro::enabled","configSlug":"gemini-2-5-pro-thinking","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (thinking)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":84,"score":145.26,"scoreLo":143.44,"scoreHi":147.35,"reasoningOffset":0.36,"rankEligible":true},{"stateId":"gemini-2-5-pro::effort:xhigh","configSlug":"gemini-2-5-pro-xhigh","modelId":"gemini-2-5-pro","modelSlug":"gemini-2-5-pro","name":"Gemini 2.5 Pro (xhigh)","modelName":"Gemini 2.5 Pro","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":145.84,"scoreLo":142.94,"scoreHi":149.06,"reasoningOffset":0.94,"rankEligible":false},{"stateId":"gemini-3-1-flash-lite::default","configSlug":"gemini-3-1-flash-lite","modelId":"gemini-3-1-flash-lite","modelSlug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite","modelName":"Gemini 3.1 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":144.26,"scoreLo":141.65,"scoreHi":147.08,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-1-flash-lite::effort:high","configSlug":"gemini-3-1-flash-lite-high","modelId":"gemini-3-1-flash-lite","modelSlug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite (high)","modelName":"Gemini 3.1 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":15,"score":145.63,"scoreLo":143.25,"scoreHi":148.23,"reasoningOffset":1.37,"rankEligible":true,"rankMean":179.4025,"rankLo":161.0,"rankHi":198.0},{"stateId":"gemini-3-1-flash-lite::effort:low","configSlug":"gemini-3-1-flash-lite-low","modelId":"gemini-3-1-flash-lite","modelSlug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite (low)","modelName":"Gemini 3.1 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":7,"score":142.01,"scoreLo":139.31,"scoreHi":144.89,"reasoningOffset":-2.24,"rankEligible":true},{"stateId":"gemini-3-1-flash-lite::effort:medium","configSlug":"gemini-3-1-flash-lite-medium","modelId":"gemini-3-1-flash-lite","modelSlug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite (medium)","modelName":"Gemini 3.1 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":3,"score":144.19,"scoreLo":140.73,"scoreHi":147.8,"reasoningOffset":-0.07,"rankEligible":false},{"stateId":"gemini-3-1-flash-lite::effort:minimal","configSlug":"gemini-3-1-flash-lite-minimal","modelId":"gemini-3-1-flash-lite","modelSlug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite (minimal)","modelName":"Gemini 3.1 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":5,"score":141.44,"scoreLo":138.55,"scoreHi":144.51,"reasoningOffset":-2.82,"rankEligible":true},{"stateId":"gemini-3-1-flash-lite-preview::default","configSlug":"gemini-3-1-flash-lite-preview","modelId":"gemini-3-1-flash-lite-preview","modelSlug":"gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash-Lite Preview","modelName":"Gemini 3.1 Flash-Lite Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":143.25,"scoreLo":141.26,"scoreHi":145.42,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-1-flash-lite-preview::effort:high","configSlug":"gemini-3-1-flash-lite-preview-high","modelId":"gemini-3-1-flash-lite-preview","modelSlug":"gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash-Lite Preview (high)","modelName":"Gemini 3.1 Flash-Lite Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":42,"score":144.74,"scoreLo":142.77,"scoreHi":146.83,"reasoningOffset":1.5,"rankEligible":true},{"stateId":"gemini-3-1-flash-lite-preview::effort:low","configSlug":"gemini-3-1-flash-lite-preview-low","modelId":"gemini-3-1-flash-lite-preview","modelSlug":"gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash-Lite Preview (low)","modelName":"Gemini 3.1 Flash-Lite Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":6,"score":140.36,"scoreLo":137.08,"scoreHi":143.73,"reasoningOffset":-2.88,"rankEligible":true},{"stateId":"gemini-3-1-flash-lite-preview::effort:medium","configSlug":"gemini-3-1-flash-lite-preview-medium","modelId":"gemini-3-1-flash-lite-preview","modelSlug":"gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash-Lite Preview (medium)","modelName":"Gemini 3.1 Flash-Lite Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":5,"score":142.04,"scoreLo":138.81,"scoreHi":145.53,"reasoningOffset":-1.21,"rankEligible":true},{"stateId":"gemini-3-1-flash-lite-preview::effort:minimal","configSlug":"gemini-3-1-flash-lite-preview-minimal","modelId":"gemini-3-1-flash-lite-preview","modelSlug":"gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash-Lite Preview (minimal)","modelName":"Gemini 3.1 Flash-Lite Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":2,"score":138.25,"scoreLo":133.71,"scoreHi":142.67,"reasoningOffset":-5.0,"rankEligible":false},{"stateId":"gemini-3-1-flash-lite-preview::enabled","configSlug":"gemini-3-1-flash-lite-preview-thinking","modelId":"gemini-3-1-flash-lite-preview","modelSlug":"gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash-Lite Preview (thinking)","modelName":"Gemini 3.1 Flash-Lite Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":15,"score":145.5,"scoreLo":143.02,"scoreHi":148.11,"reasoningOffset":2.25,"rankEligible":true,"rankMean":180.7361,"rankLo":161.0,"rankHi":200.0},{"stateId":"gemini-3-1-flash-lite::enabled","configSlug":"gemini-3-1-flash-lite-thinking","modelId":"gemini-3-1-flash-lite","modelSlug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite (thinking)","modelName":"Gemini 3.1 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":12,"score":144.55,"scoreLo":141.56,"scoreHi":147.69,"reasoningOffset":0.29,"rankEligible":true},{"stateId":"gemini-3-1-pro-preview::enabled","configSlug":"gemini-3-1-pro","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (enabled)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":121,"score":156.36,"scoreLo":153.6,"scoreHi":159.75,"reasoningOffset":2.81,"rankEligible":true,"rankMean":77.5497,"rankLo":74.0,"rankHi":81.0},{"stateId":"gemini-3-1-pro-preview::effort:65535","configSlug":"gemini-3-1-pro-65535","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (65535)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"65535","reasoningCondition":"effort:65535","isDefault":false,"n":1,"score":154.07,"scoreLo":149.36,"scoreHi":159.16,"reasoningOffset":0.52,"rankEligible":false},{"stateId":"gemini-3-1-pro-preview::effort:high","configSlug":"gemini-3-1-pro-high","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (high)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":169,"score":156.21,"scoreLo":153.44,"scoreHi":159.46,"reasoningOffset":2.66,"rankEligible":true},{"stateId":"gemini-3-1-pro-preview::effort:low","configSlug":"gemini-3-1-pro-low","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (low)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":16,"score":153.54,"scoreLo":150.59,"scoreHi":157.01,"reasoningOffset":-0.01,"rankEligible":true},{"stateId":"gemini-3-1-pro-preview::effort:medium","configSlug":"gemini-3-1-pro-medium","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (medium)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":16,"score":155.15,"scoreLo":152.17,"scoreHi":158.65,"reasoningOffset":1.6,"rankEligible":true},{"stateId":"gemini-3-1-pro-preview::default","configSlug":"gemini-3-1-pro-preview","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":57,"score":153.55,"scoreLo":151.0,"scoreHi":156.55,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-1-pro-preview::effort:1k","configSlug":"gemini-3-1-pro-preview-1k","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (1k)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"1k","reasoningCondition":"effort:1k","isDefault":false,"n":1,"score":152.02,"scoreLo":149.07,"scoreHi":155.5,"reasoningOffset":-1.53,"rankEligible":false},{"stateId":"gemini-3-1-pro-preview::effort:max","configSlug":"gemini-3-1-pro-preview-max-reactbench-v1-gemini-cli","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (max)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":2,"score":154.47,"scoreLo":151.28,"scoreHi":158.08,"reasoningOffset":0.92,"rankEligible":false},{"stateId":"gemini-3-1-pro-preview::effort:minimal","configSlug":"gemini-3-1-pro-preview-minimal","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (minimal)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":2,"score":149.1,"scoreLo":144.51,"scoreHi":154.14,"reasoningOffset":-4.46,"rankEligible":false},{"stateId":"gemini-3-1-pro-preview::effort:xhigh","configSlug":"gemini-3-1-pro-preview-xhigh-reactbench-v1-gemini-cli","modelId":"gemini-3-1-pro-preview","modelSlug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview (xhigh)","modelName":"Gemini 3.1 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":154.57,"scoreLo":151.3,"scoreHi":158.28,"reasoningOffset":1.02,"rankEligible":false},{"stateId":"gemini-3-5-flash::default","configSlug":"gemini-3-5-flash","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","modelName":"Gemini 3.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":55,"score":156.29,"scoreLo":153.5,"scoreHi":159.66,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-5-flash::effort:high","configSlug":"gemini-3-5-flash-high","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash (high)","modelName":"Gemini 3.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":133,"score":157.18,"scoreLo":154.31,"scoreHi":160.67,"reasoningOffset":0.9,"rankEligible":true},{"stateId":"gemini-3-5-flash-lite::effort:minimal","configSlug":"gemini-3-5-flash-lite","modelId":"gemini-3-5-flash-lite","modelSlug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite (minimal)","modelName":"Gemini 3.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":true,"n":78,"score":145.48,"scoreLo":143.53,"scoreHi":147.68,"reasoningOffset":-4.53,"rankEligible":true},{"stateId":"gemini-3-5-flash-lite::enabled","configSlug":"gemini-3-5-flash-lite-artificial-analysis","modelId":"gemini-3-5-flash-lite","modelSlug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite (enabled)","modelName":"Gemini 3.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":16,"score":151.22,"scoreLo":148.46,"scoreHi":154.46,"reasoningOffset":1.21,"rankEligible":true,"rankMean":118.6586,"rankLo":102.0,"rankHi":138.0},{"stateId":"gemini-3-5-flash-lite::effort:high","configSlug":"gemini-3-5-flash-lite-high","modelId":"gemini-3-5-flash-lite","modelSlug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite (high)","modelName":"Gemini 3.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":66,"score":149.62,"scoreLo":147.38,"scoreHi":152.21,"reasoningOffset":-0.39,"rankEligible":true},{"stateId":"gemini-3-5-flash-lite::effort:low","configSlug":"gemini-3-5-flash-lite-low","modelId":"gemini-3-5-flash-lite","modelSlug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite (low)","modelName":"Gemini 3.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":12,"score":145.95,"scoreLo":143.62,"scoreHi":148.5,"reasoningOffset":-4.06,"rankEligible":true},{"stateId":"gemini-3-5-flash-lite::effort:medium","configSlug":"gemini-3-5-flash-lite-medium","modelId":"gemini-3-5-flash-lite","modelSlug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite (medium)","modelName":"Gemini 3.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":6,"score":147.82,"scoreLo":145.39,"scoreHi":150.62,"reasoningOffset":-2.19,"rankEligible":true},{"stateId":"gemini-3-5-flash-lite::default","configSlug":"gemini-3-5-flash-lite-unspecified","modelId":"gemini-3-5-flash-lite","modelSlug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","modelName":"Gemini 3.5 Flash-Lite","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":5,"score":150.01,"scoreLo":147.53,"scoreHi":152.78,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-5-flash::effort:low","configSlug":"gemini-3-5-flash-low","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash (low)","modelName":"Gemini 3.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":19,"score":155.12,"scoreLo":152.06,"scoreHi":158.73,"reasoningOffset":-1.17,"rankEligible":true},{"stateId":"gemini-3-5-flash::effort:medium","configSlug":"gemini-3-5-flash-medium","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash (medium)","modelName":"Gemini 3.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":31,"score":156.85,"scoreLo":153.87,"scoreHi":160.41,"reasoningOffset":0.56,"rankEligible":true},{"stateId":"gemini-3-5-flash::effort:minimal","configSlug":"gemini-3-5-flash-minimal","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash (minimal)","modelName":"Gemini 3.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":18,"score":152.07,"scoreLo":149.5,"scoreHi":155.13,"reasoningOffset":-4.22,"rankEligible":true},{"stateId":"gemini-3-5-flash::enabled","configSlug":"gemini-3-5-flash-pi","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash (enabled)","modelName":"Gemini 3.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":23,"score":158.42,"scoreLo":155.27,"scoreHi":162.2,"reasoningOffset":2.14,"rankEligible":true,"rankMean":64.8731,"rankLo":52.0,"rankHi":73.0},{"stateId":"gemini-3-5-flash::effort:xhigh","configSlug":"gemini-3-5-flash-xhigh-reactbench-v1-gemini-cli","modelId":"gemini-3-5-flash","modelSlug":"gemini-3-5-flash","name":"Gemini 3.5 Flash (xhigh)","modelName":"Gemini 3.5 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":156.59,"scoreLo":153.25,"scoreHi":160.48,"reasoningOffset":0.3,"rankEligible":false},{"stateId":"gemini-3-6-flash::effort:medium","configSlug":"gemini-3-6-flash","modelId":"gemini-3-6-flash","modelSlug":"gemini-3-6-flash","name":"Gemini 3.6 Flash (medium)","modelName":"Gemini 3.6 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":true,"n":38,"score":157.38,"scoreLo":154.42,"scoreHi":161.05,"reasoningOffset":1.24,"rankEligible":true,"rankMean":72.2828,"rankLo":66.0,"rankHi":78.0},{"stateId":"gemini-3-6-flash::effort:high","configSlug":"gemini-3-6-flash-high","modelId":"gemini-3-6-flash","modelSlug":"gemini-3-6-flash","name":"Gemini 3.6 Flash (high)","modelName":"Gemini 3.6 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":110,"score":157.08,"scoreLo":154.21,"scoreHi":160.53,"reasoningOffset":0.94,"rankEligible":true},{"stateId":"gemini-3-6-flash::effort:low","configSlug":"gemini-3-6-flash-low","modelId":"gemini-3-6-flash","modelSlug":"gemini-3-6-flash","name":"Gemini 3.6 Flash (low)","modelName":"Gemini 3.6 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":16,"score":154.78,"scoreLo":151.94,"scoreHi":158.16,"reasoningOffset":-1.36,"rankEligible":true},{"stateId":"gemini-3-6-flash::effort:minimal","configSlug":"gemini-3-6-flash-minimal","modelId":"gemini-3-6-flash","modelSlug":"gemini-3-6-flash","name":"Gemini 3.6 Flash (minimal)","modelName":"Gemini 3.6 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":12,"score":151.2,"scoreLo":148.66,"scoreHi":154.07,"reasoningOffset":-4.93,"rankEligible":true},{"stateId":"gemini-3-6-flash::default","configSlug":"gemini-3-6-flash-unspecified","modelId":"gemini-3-6-flash","modelSlug":"gemini-3-6-flash","name":"Gemini 3.6 Flash","modelName":"Gemini 3.6 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":156.14,"scoreLo":153.3,"scoreHi":159.57,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-7-flash::default","configSlug":"gemini-3-7-flash","modelId":"gemini-3-7-flash","modelSlug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","modelName":"Gemini 3.7 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":60,"score":159.5,"scoreLo":156.38,"scoreHi":163.22,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-7-flash::effort:high","configSlug":"gemini-3-7-flash-high","modelId":"gemini-3-7-flash","modelSlug":"gemini-3-7-flash","name":"Gemini 3.7 Flash (high)","modelName":"Gemini 3.7 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":120,"score":160.61,"scoreLo":157.36,"scoreHi":164.54,"reasoningOffset":1.11,"rankEligible":true,"rankMean":42.6103,"rankLo":36.0,"rankHi":48.0},{"stateId":"gemini-3-7-flash::effort:low","configSlug":"gemini-3-7-flash-low","modelId":"gemini-3-7-flash","modelSlug":"gemini-3-7-flash","name":"Gemini 3.7 Flash (low)","modelName":"Gemini 3.7 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":18,"score":158.55,"scoreLo":155.44,"scoreHi":162.35,"reasoningOffset":-0.95,"rankEligible":true},{"stateId":"gemini-3-7-flash::effort:medium","configSlug":"gemini-3-7-flash-medium","modelId":"gemini-3-7-flash","modelSlug":"gemini-3-7-flash","name":"Gemini 3.7 Flash (medium)","modelName":"Gemini 3.7 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":26,"score":159.93,"scoreLo":156.69,"scoreHi":163.86,"reasoningOffset":0.43,"rankEligible":true},{"stateId":"gemini-3-7-flash::enabled","configSlug":"gemini-3-7-flash-proximus","modelId":"gemini-3-7-flash","modelSlug":"gemini-3-7-flash","name":"Gemini 3.7 Flash (enabled)","modelName":"Gemini 3.7 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":1,"score":160.54,"scoreLo":156.48,"scoreHi":165.23,"reasoningOffset":1.04,"rankEligible":false},{"stateId":"gemini-3-8-flash::default","configSlug":"gemini-3-8-flash","modelId":"gemini-3-8-flash","modelSlug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","modelName":"Gemini 3.8 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":81,"score":160.17,"scoreLo":156.95,"scoreHi":164.02,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-8-flash::enabled","configSlug":"gemini-3-8-flash-concept-synth-cascade","modelId":"gemini-3-8-flash","modelSlug":"gemini-3-8-flash","name":"Gemini 3.8 Flash (enabled)","modelName":"Gemini 3.8 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":159.97,"scoreLo":156.24,"scoreHi":164.38,"reasoningOffset":-0.19,"rankEligible":false},{"stateId":"gemini-3-8-flash::effort:high","configSlug":"gemini-3-8-flash-high","modelId":"gemini-3-8-flash","modelSlug":"gemini-3-8-flash","name":"Gemini 3.8 Flash (high)","modelName":"Gemini 3.8 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":172,"score":161.12,"scoreLo":157.83,"scoreHi":165.08,"reasoningOffset":0.95,"rankEligible":true},{"stateId":"gemini-3-8-flash::effort:low","configSlug":"gemini-3-8-flash-low","modelId":"gemini-3-8-flash","modelSlug":"gemini-3-8-flash","name":"Gemini 3.8 Flash (low)","modelName":"Gemini 3.8 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":30,"score":159.15,"scoreLo":155.99,"scoreHi":162.89,"reasoningOffset":-1.02,"rankEligible":true},{"stateId":"gemini-3-8-flash::effort:medium","configSlug":"gemini-3-8-flash-medium","modelId":"gemini-3-8-flash","modelSlug":"gemini-3-8-flash","name":"Gemini 3.8 Flash (medium)","modelName":"Gemini 3.8 Flash","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":38,"score":161.78,"scoreLo":158.27,"scoreHi":165.87,"reasoningOffset":1.61,"rankEligible":true,"rankMean":29.9931,"rankLo":23.0,"rankHi":38.0},{"stateId":"gemini-3-deep-think::default","configSlug":"gemini-3-deep-think","modelId":"gemini-3-deep-think","modelSlug":"gemini-3-deep-think","name":"Gemini 3 Deep Think","modelName":"Gemini 3 Deep Think","organizationSlug":"google","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":161.79,"scoreLo":157.75,"scoreHi":166.61,"reasoningOffset":2.4,"rankEligible":true,"rankMean":31.2811,"rankLo":18.0,"rankHi":53.0},{"stateId":"gemini-3-flash-preview::enabled","configSlug":"gemini-3-flash","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview (enabled)","modelName":"Gemini 3 Flash Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":77,"score":152.57,"scoreLo":150.06,"scoreHi":155.44,"reasoningOffset":2.22,"rankEligible":true,"rankMean":105.8397,"rankLo":98.0,"rankHi":114.0},{"stateId":"gemini-3-flash-preview::effort:high","configSlug":"gemini-3-flash-high","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview (high)","modelName":"Gemini 3 Flash Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":70,"score":152.4,"scoreLo":149.91,"scoreHi":155.19,"reasoningOffset":2.06,"rankEligible":true},{"stateId":"gemini-3-flash-preview::effort:low","configSlug":"gemini-3-flash-low","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview (low)","modelName":"Gemini 3 Flash Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":10,"score":147.49,"scoreLo":145.08,"scoreHi":150.02,"reasoningOffset":-2.85,"rankEligible":true},{"stateId":"gemini-3-flash-preview::effort:medium","configSlug":"gemini-3-flash-medium","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview (medium)","modelName":"Gemini 3 Flash Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":9,"score":150.45,"scoreLo":148.0,"scoreHi":153.3,"reasoningOffset":0.1,"rankEligible":true},{"stateId":"gemini-3-flash-preview::effort:minimal","configSlug":"gemini-3-flash-minimal","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview (minimal)","modelName":"Gemini 3 Flash Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":15,"score":147.02,"scoreLo":144.75,"scoreHi":149.45,"reasoningOffset":-3.33,"rankEligible":true},{"stateId":"gemini-3-flash-preview::disabled","configSlug":"gemini-3-flash-no-thinking","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview (disabled)","modelName":"Gemini 3 Flash Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":10,"score":149.3,"scoreLo":146.37,"scoreHi":152.72,"reasoningOffset":-1.05,"rankEligible":true},{"stateId":"gemini-3-flash-preview::default","configSlug":"gemini-3-flash-preview","modelId":"gemini-3-flash-preview","modelSlug":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview","modelName":"Gemini 3 Flash Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":13,"score":150.35,"scoreLo":147.99,"scoreHi":153.04,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-pro-preview::enabled","configSlug":"gemini-3-pro","modelId":"gemini-3-pro-preview","modelSlug":"gemini-3-pro-preview","name":"Gemini 3 Pro Preview (enabled)","modelName":"Gemini 3 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":90,"score":153.9,"scoreLo":151.35,"scoreHi":156.94,"reasoningOffset":1.19,"rankEligible":true},{"stateId":"gemini-3-pro-preview::effort:high","configSlug":"gemini-3-pro-high","modelId":"gemini-3-pro-preview","modelSlug":"gemini-3-pro-preview","name":"Gemini 3 Pro Preview (high)","modelName":"Gemini 3 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":81,"score":154.17,"scoreLo":151.59,"scoreHi":157.25,"reasoningOffset":1.46,"rankEligible":true,"rankMean":91.2183,"rankLo":85.0,"rankHi":99.0},{"stateId":"gemini-3-pro-preview::effort:low","configSlug":"gemini-3-pro-low","modelId":"gemini-3-pro-preview","modelSlug":"gemini-3-pro-preview","name":"Gemini 3 Pro Preview (low)","modelName":"Gemini 3 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":10,"score":150.9,"scoreLo":147.66,"scoreHi":154.59,"reasoningOffset":-1.81,"rankEligible":true},{"stateId":"gemini-3-pro-preview::default","configSlug":"gemini-3-pro-preview","modelId":"gemini-3-pro-preview","modelSlug":"gemini-3-pro-preview","name":"Gemini 3 Pro Preview","modelName":"Gemini 3 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":89,"score":152.71,"scoreLo":150.16,"scoreHi":155.61,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-3-pro-preview::effort:minimal","configSlug":"gemini-3-pro-preview-minimal","modelId":"gemini-3-pro-preview","modelSlug":"gemini-3-pro-preview","name":"Gemini 3 Pro Preview (minimal)","modelName":"Gemini 3 Pro Preview","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":148.61,"scoreLo":143.84,"scoreHi":153.79,"reasoningOffset":-4.1,"rankEligible":false},{"stateId":"gemini-4-argon::default","configSlug":"gemini-4-argon","modelId":"gemini-4-argon","modelSlug":"gemini-4-argon","name":"Gemini 4 Argon","modelName":"Gemini 4 Argon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":47,"score":165.42,"scoreLo":161.58,"scoreHi":169.99,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemini-4-argon::enabled","configSlug":"gemini-4-argon-concept-synth-cascade","modelId":"gemini-4-argon","modelSlug":"gemini-4-argon","name":"Gemini 4 Argon (enabled)","modelName":"Gemini 4 Argon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":167.0,"scoreLo":162.62,"scoreHi":172.25,"reasoningOffset":1.57,"rankEligible":false},{"stateId":"gemini-4-argon::effort:high","configSlug":"gemini-4-argon-high","modelId":"gemini-4-argon","modelSlug":"gemini-4-argon","name":"Gemini 4 Argon (high)","modelName":"Gemini 4 Argon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":140,"score":167.01,"scoreLo":163.04,"scoreHi":171.73,"reasoningOffset":1.59,"rankEligible":true,"rankMean":7.1197,"rankLo":5.0,"rankHi":9.0},{"stateId":"gemini-4-argon::effort:low","configSlug":"gemini-4-argon-low","modelId":"gemini-4-argon","modelSlug":"gemini-4-argon","name":"Gemini 4 Argon (low)","modelName":"Gemini 4 Argon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":14,"score":162.95,"scoreLo":159.29,"scoreHi":167.3,"reasoningOffset":-2.48,"rankEligible":true},{"stateId":"gemini-4-argon::effort:medium","configSlug":"gemini-4-argon-medium","modelId":"gemini-4-argon","modelSlug":"gemini-4-argon","name":"Gemini 4 Argon (medium)","modelName":"Gemini 4 Argon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":18,"score":166.51,"scoreLo":162.5,"scoreHi":171.44,"reasoningOffset":1.09,"rankEligible":true},{"stateId":"gemini-4-carbon::default","configSlug":"gemini-4-carbon-andon-blueprint-bench-2","modelId":"gemini-4-carbon","modelSlug":"gemini-4-carbon","name":"Gemini 4 Carbon","modelName":"Gemini 4 Carbon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":1,"score":167.09,"scoreLo":163.01,"scoreHi":172.08,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gemini-4-carbon::effort:high","configSlug":"gemini-4-carbon-high","modelId":"gemini-4-carbon","modelSlug":"gemini-4-carbon","name":"Gemini 4 Carbon (high)","modelName":"Gemini 4 Carbon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":27,"score":168.62,"scoreLo":164.37,"scoreHi":173.84,"reasoningOffset":1.52,"rankEligible":true,"rankMean":2.6239,"rankLo":1.0,"rankHi":4.0},{"stateId":"gemini-4-carbon::effort:low","configSlug":"gemini-4-carbon-low","modelId":"gemini-4-carbon","modelSlug":"gemini-4-carbon","name":"Gemini 4 Carbon (low)","modelName":"Gemini 4 Carbon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":8,"score":166.76,"scoreLo":162.52,"scoreHi":171.71,"reasoningOffset":-0.34,"rankEligible":true},{"stateId":"gemini-4-carbon::effort:medium","configSlug":"gemini-4-carbon-medium","modelId":"gemini-4-carbon","modelSlug":"gemini-4-carbon","name":"Gemini 4 Carbon (medium)","modelName":"Gemini 4 Carbon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":9,"score":167.89,"scoreLo":163.65,"scoreHi":173.0,"reasoningOffset":0.8,"rankEligible":true},{"stateId":"gemini-4-carbon::enabled","configSlug":"gemini-4-carbon-proximus","modelId":"gemini-4-carbon","modelSlug":"gemini-4-carbon","name":"Gemini 4 Carbon (enabled)","modelName":"Gemini 4 Carbon","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":1,"score":168.01,"scoreLo":163.2,"scoreHi":173.75,"reasoningOffset":0.92,"rankEligible":false},{"stateId":"gemini-exp-1121::default","configSlug":"gemini-exp-1121","modelId":"gemini-exp-1121","modelSlug":"gemini-exp-1121","name":"Gemini Experimental 1121","modelName":"Gemini Experimental 1121","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":136.08,"scoreLo":127.8,"scoreHi":144.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":294.2008,"rankLo":184.0,"rankHi":404.0},{"stateId":"gemini-exp-1206::default","configSlug":"gemini-exp-1206","modelId":"gemini-exp-1206","modelSlug":"gemini-exp-1206","name":"Gemini Exp-1206","modelName":"Gemini Exp-1206","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":136.9,"scoreLo":132.3,"scoreHi":141.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":282.2356,"rankLo":224.0,"rankHi":341.025},{"stateId":"gemini-nano-1::default","configSlug":"gemini-nano-1","modelId":"gemini-nano-1","modelSlug":"gemini-nano-1","name":"Gemini Nano 1","modelName":"Gemini Nano 1","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":76.99,"scoreLo":65.09,"scoreHi":86.59,"reasoningOffset":0.0,"rankEligible":true,"rankMean":842.5767,"rankLo":821.0,"rankHi":861.0},{"stateId":"gemini-nano-2::default","configSlug":"gemini-nano-2","modelId":"gemini-nano-2","modelSlug":"gemini-nano-2","name":"Gemini Nano 2","modelName":"Gemini Nano 2","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":84.95,"scoreLo":75.2,"scoreHi":93.07,"reasoningOffset":0.0,"rankEligible":true,"rankMean":814.0189,"rankLo":791.0,"rankHi":835.0},{"stateId":"gemini-pro::default","configSlug":"gemini-pro","modelId":"gemini-pro","modelSlug":"gemini-pro","name":"Gemini 1.0 Pro","modelName":"Gemini 1.0 Pro","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":109.03,"scoreLo":103.97,"scoreHi":113.27,"reasoningOffset":0.0,"rankEligible":true,"rankMean":634.6358,"rankLo":606.0,"rankHi":662.0},{"stateId":"gemini-pro-vision::default","configSlug":"gemini-pro-vision","modelId":"gemini-pro-vision","modelSlug":"gemini-pro-vision","name":"Gemini 1.0 Pro Vision","modelName":"Gemini 1.0 Pro Vision","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":106.6,"scoreLo":98.54,"scoreHi":113.24,"reasoningOffset":0.0,"rankEligible":true,"rankMean":658.9736,"rankLo":593.0,"rankHi":718.0},{"stateId":"gemini-robotics-er-1-5-preview::default","configSlug":"gemini-robotics-er-1-5-preview","modelId":"gemini-robotics-er-1-5-preview","modelSlug":"gemini-robotics-er-1-5-preview","name":"Gemini Robotics-ER 1.5","modelName":"Gemini Robotics-ER 1.5","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":3,"score":145.68,"scoreLo":140.8,"scoreHi":150.46,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gemini-robotics-er-1-5-preview::disabled","configSlug":"gemini-robotics-er-1-5-preview-no-thinking","modelId":"gemini-robotics-er-1-5-preview","modelSlug":"gemini-robotics-er-1-5-preview","name":"Gemini Robotics-ER 1.5 (disabled)","modelName":"Gemini Robotics-ER 1.5","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":4,"score":144.42,"scoreLo":140.68,"scoreHi":148.29,"reasoningOffset":-1.27,"rankEligible":true},{"stateId":"gemini-robotics-er-1-5-preview::enabled","configSlug":"gemini-robotics-er-1-5-preview-thinking","modelId":"gemini-robotics-er-1-5-preview","modelSlug":"gemini-robotics-er-1-5-preview","name":"Gemini Robotics-ER 1.5 (thinking)","modelName":"Gemini Robotics-ER 1.5","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":4,"score":146.5,"scoreLo":142.48,"scoreHi":150.62,"reasoningOffset":0.82,"rankEligible":true,"rankMean":169.9358,"rankLo":130.0,"rankHi":209.0},{"stateId":"gemini-ultra::default","configSlug":"gemini-ultra","modelId":"gemini-ultra","modelSlug":"gemini-ultra","name":"Gemini 1.0 Ultra","modelName":"Gemini 1.0 Ultra","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":120.72,"scoreLo":116.03,"scoreHi":125.06,"reasoningOffset":0.0,"rankEligible":true,"rankMean":492.7144,"rankLo":439.0,"rankHi":541.0},{"stateId":"gemma-1-1-2b-it::default","configSlug":"gemma-1-1-2b-it","modelId":"gemma-1-1-2b-it","modelSlug":"gemma-1-1-2b-it","name":"Gemma 1.1 2B IT","modelName":"Gemma 1.1 2B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":76.98,"scoreLo":65.18,"scoreHi":86.84,"reasoningOffset":0.0,"rankEligible":true,"rankMean":842.6344,"rankLo":821.0,"rankHi":860.0},{"stateId":"gemma-1-1-7b-it::default","configSlug":"gemma-1-1-7b-it","modelId":"gemma-1-1-7b-it","modelSlug":"gemma-1-1-7b-it","name":"Gemma 1.1 7B IT","modelName":"Gemma 1.1 7B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":95.45,"scoreLo":88.81,"scoreHi":100.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":755.6581,"rankLo":741.0,"rankHi":769.0},{"stateId":"gemma-2-27b::default","configSlug":"gemma-2-27b","modelId":"gemma-2-27b","modelSlug":"gemma-2-27b","name":"Gemma 2 27B","modelName":"Gemma 2 27B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":111.28,"scoreLo":106.58,"scoreHi":115.34,"reasoningOffset":0.0,"rankEligible":true,"rankMean":608.0958,"rankLo":578.0,"rankHi":637.0},{"stateId":"gemma-2-27b-it::default","configSlug":"gemma-2-27b-it","modelId":"gemma-2-27b-it","modelSlug":"gemma-2-27b-it","name":"Gemma 2 27B IT","modelName":"Gemma 2 27B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":41,"score":117.31,"scoreLo":113.71,"scoreHi":120.51,"reasoningOffset":0.0,"rankEligible":true,"rankMean":535.1264,"rankLo":514.0,"rankHi":555.0},{"stateId":"gemma-2-2b::default","configSlug":"gemma-2-2b","modelId":"gemma-2-2b","modelSlug":"gemma-2-2b","name":"Gemma 2 2B","modelName":"Gemma 2 2B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":82.92,"scoreLo":73.7,"scoreHi":90.53,"reasoningOffset":0.0,"rankEligible":true,"rankMean":822.5347,"rankLo":807.0,"rankHi":839.0},{"stateId":"gemma-2-2b-it::default","configSlug":"gemma-2-2b-it","modelId":"gemma-2-2b-it","modelSlug":"gemma-2-2b-it","name":"Gemma 2 2B IT","modelName":"Gemma 2 2B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":94.2,"scoreLo":87.08,"scoreHi":100.15,"reasoningOffset":0.0,"rankEligible":true,"rankMean":764.285,"rankLo":743.0,"rankHi":783.0},{"stateId":"gemma-2-9b::default","configSlug":"gemma-2-9b","modelId":"gemma-2-9b","modelSlug":"gemma-2-9b","name":"Gemma 2 9B","modelName":"Gemma 2 9B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":107.14,"scoreLo":101.9,"scoreHi":111.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":655.375,"rankLo":629.0,"rankHi":680.0},{"stateId":"gemma-2-9b-it::default","configSlug":"gemma-2-9b-it","modelId":"gemma-2-9b-it","modelSlug":"gemma-2-9b-it","name":"Gemma 2 9B IT","modelName":"Gemma 2 9B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":39,"score":111.21,"scoreLo":106.93,"scoreHi":114.89,"reasoningOffset":0.0,"rankEligible":true,"rankMean":609.0633,"rankLo":585.0,"rankHi":632.0},{"stateId":"gemma-2b::default","configSlug":"gemma-2b","modelId":"gemma-2b","modelSlug":"gemma-2b","name":"Gemma 2B","modelName":"Gemma 2B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":40,"score":74.75,"scoreLo":64.66,"scoreHi":82.9,"reasoningOffset":0.0,"rankEligible":true,"rankMean":849.7086,"rankLo":841.0,"rankHi":858.0},{"stateId":"gemma-2b-it::default","configSlug":"gemma-2b-it","modelId":"gemma-2b-it","modelSlug":"gemma-2b-it","name":"Gemma 2B IT","modelName":"Gemma 2B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":74.77,"scoreLo":64.46,"scoreHi":83.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":849.4767,"rankLo":837.0,"rankHi":861.0},{"stateId":"gemma-3-12b-it::default","configSlug":"gemma-3-12b-it","modelId":"gemma-3-12b-it","modelSlug":"gemma-3-12b-it","name":"Gemma 3 12B IT","modelName":"Gemma 3 12B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":75,"score":119.6,"scoreLo":116.31,"scoreHi":122.43,"reasoningOffset":0.0,"rankEligible":true,"rankMean":508.6161,"rankLo":487.0,"rankHi":528.025},{"stateId":"gemma-3-12b-pt::default","configSlug":"gemma-3-12b-pt","modelId":"gemma-3-12b-pt","modelSlug":"gemma-3-12b-pt","name":"Gemma 3 12B PT","modelName":"Gemma 3 12B PT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":108.24,"scoreLo":102.91,"scoreHi":112.8,"reasoningOffset":0.0,"rankEligible":true,"rankMean":643.4017,"rankLo":613.0,"rankHi":674.0},{"stateId":"gemma-3-1b-it::default","configSlug":"gemma-3-1b-it","modelId":"gemma-3-1b-it","modelSlug":"gemma-3-1b-it","name":"Gemma 3 1B IT","modelName":"Gemma 3 1B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":34,"score":84.07,"scoreLo":74.9,"scoreHi":91.69,"reasoningOffset":0.0,"rankEligible":true,"rankMean":817.9603,"rankLo":802.0,"rankHi":834.0},{"stateId":"gemma-3-1b-pt::default","configSlug":"gemma-3-1b-pt","modelId":"gemma-3-1b-pt","modelSlug":"gemma-3-1b-pt","name":"Gemma 3 1B PT","modelName":"Gemma 3 1B PT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":62.32,"scoreLo":47.72,"scoreHi":74.09,"reasoningOffset":0.0,"rankEligible":true,"rankMean":869.7122,"rankLo":861.0,"rankHi":877.025},{"stateId":"gemma-3-270m::default","configSlug":"gemma-3-270m","modelId":"gemma-3-270m","modelSlug":"gemma-3-270m","name":"Gemma 3 270M","modelName":"Gemma 3 270M","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":46.31,"scoreLo":27.66,"scoreHi":62.05,"reasoningOffset":0.0,"rankEligible":true,"rankMean":881.4578,"rankLo":875.0,"rankHi":885.0},{"stateId":"gemma-3-270m-it::default","configSlug":"gemma-3-270m-it","modelId":"gemma-3-270m-it","modelSlug":"gemma-3-270m-it","name":"Gemma 3 270M IT","modelName":"Gemma 3 270M IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":49.2,"scoreLo":28.71,"scoreHi":67.22,"reasoningOffset":0.0,"rankEligible":true,"rankMean":879.5881,"rankLo":868.0,"rankHi":885.0},{"stateId":"gemma-3-27b-it::default","configSlug":"gemma-3-27b-it","modelId":"gemma-3-27b-it","modelSlug":"gemma-3-27b-it","name":"Gemma 3 27B IT","modelName":"Gemma 3 27B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":92,"score":123.93,"scoreLo":121.32,"scoreHi":126.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":449.0842,"rankLo":431.0,"rankHi":468.0},{"stateId":"gemma-3-27b-pt::default","configSlug":"gemma-3-27b-pt","modelId":"gemma-3-27b-pt","modelSlug":"gemma-3-27b-pt","name":"Gemma 3 27B PT","modelName":"Gemma 3 27B PT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":114.83,"scoreLo":110.3,"scoreHi":118.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":563.9228,"rankLo":534.0,"rankHi":596.0},{"stateId":"gemma-3-4b-it::default","configSlug":"gemma-3-4b-it","modelId":"gemma-3-4b-it","modelSlug":"gemma-3-4b-it","name":"Gemma 3 4B IT","modelName":"Gemma 3 4B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":55,"score":110.47,"scoreLo":105.8,"scoreHi":114.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":617.8811,"rankLo":590.0,"rankHi":644.0},{"stateId":"gemma-3-4b-pt::default","configSlug":"gemma-3-4b-pt","modelId":"gemma-3-4b-pt","modelSlug":"gemma-3-4b-pt","name":"Gemma 3 4B PT","modelName":"Gemma 3 4B PT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":92.7,"scoreLo":84.87,"scoreHi":98.93,"reasoningOffset":0.0,"rankEligible":true,"rankMean":773.9069,"rankLo":752.0,"rankHi":795.0},{"stateId":"gemma-3n-e2b::default","configSlug":"gemma-3n-e2b","modelId":"gemma-3n-e2b","modelSlug":"gemma-3n-e2b","name":"Gemma 3n E2B","modelName":"Gemma 3n E2B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":87.24,"scoreLo":77.16,"scoreHi":95.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":803.4406,"rankLo":771.0,"rankHi":830.0},{"stateId":"gemma-3n-e2b-it::default","configSlug":"gemma-3n-e2b-it","modelId":"gemma-3n-e2b-it","modelSlug":"gemma-3n-e2b-it","name":"Gemma 3n E2B Instruct","modelName":"Gemma 3n E2B Instruct","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":99.24,"scoreLo":91.26,"scoreHi":105.94,"reasoningOffset":0.0,"rankEligible":true,"rankMean":724.5833,"rankLo":683.0,"rankHi":768.0},{"stateId":"gemma-3n-e4b::default","configSlug":"gemma-3n-e4b","modelId":"gemma-3n-e4b","modelSlug":"gemma-3n-e4b","name":"Gemma 3n E4B","modelName":"Gemma 3n E4B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":96.44,"scoreLo":88.35,"scoreHi":103.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":746.5964,"rankLo":705.0,"rankHi":784.0},{"stateId":"gemma-3n-e4b-it::default","configSlug":"gemma-3n-e4b-it","modelId":"gemma-3n-e4b-it","modelSlug":"gemma-3n-e4b-it","name":"Gemma 3n E4B Instruct","modelName":"Gemma 3n E4B Instruct","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":109.17,"scoreLo":103.83,"scoreHi":113.77,"reasoningOffset":0.0,"rankEligible":true,"rankMean":632.8608,"rankLo":593.975,"rankHi":669.0},{"stateId":"gemma-4-12b-it::default","configSlug":"gemma-4-12b-it","modelId":"gemma-4-12b-it","modelSlug":"gemma-4-12b-it","name":"Gemma 4 12B IT","modelName":"Gemma 4 12B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":139.06,"scoreLo":136.16,"scoreHi":141.87,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gemma-4-12b-it::disabled","configSlug":"gemma-4-12b-it-no-thinking","modelId":"gemma-4-12b-it","modelSlug":"gemma-4-12b-it","name":"Gemma 4 12B IT (disabled)","modelName":"Gemma 4 12B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":136.19,"scoreLo":132.46,"scoreHi":139.83,"reasoningOffset":-2.87,"rankEligible":true},{"stateId":"gemma-4-12b-it::enabled","configSlug":"gemma-4-12b-it-thinking","modelId":"gemma-4-12b-it","modelSlug":"gemma-4-12b-it","name":"Gemma 4 12B IT (thinking)","modelName":"Gemma 4 12B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":69,"score":140.72,"scoreLo":138.59,"scoreHi":142.82,"reasoningOffset":1.65,"rankEligible":true,"rankMean":230.5536,"rankLo":213.0,"rankHi":252.0},{"stateId":"gemma-4-26b-a4b-it::default","configSlug":"gemma-4-26b-a4b-it","modelId":"gemma-4-26b-a4b-it","modelSlug":"gemma-4-26b-a4b-it","name":"Gemma 4 26B A4B IT","modelName":"Gemma 4 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":48,"score":142.58,"scoreLo":140.8,"scoreHi":144.63,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemma-4-26b-a4b-it::effort:high","configSlug":"gemma-4-26b-a4b-it-high","modelId":"gemma-4-26b-a4b-it","modelSlug":"gemma-4-26b-a4b-it","name":"Gemma 4 26B A4B IT (high)","modelName":"Gemma 4 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":144.02,"scoreLo":141.48,"scoreHi":146.93,"reasoningOffset":1.44,"rankEligible":false},{"stateId":"gemma-4-26b-a4b-it::effort:low","configSlug":"gemma-4-26b-a4b-it-low","modelId":"gemma-4-26b-a4b-it","modelSlug":"gemma-4-26b-a4b-it","name":"Gemma 4 26B A4B IT (low)","modelName":"Gemma 4 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":142.24,"scoreLo":138.52,"scoreHi":146.22,"reasoningOffset":-0.35,"rankEligible":false},{"stateId":"gemma-4-26b-a4b-it::effort:medium","configSlug":"gemma-4-26b-a4b-it-medium","modelId":"gemma-4-26b-a4b-it","modelSlug":"gemma-4-26b-a4b-it","name":"Gemma 4 26B A4B IT (medium)","modelName":"Gemma 4 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":143.23,"scoreLo":139.82,"scoreHi":146.91,"reasoningOffset":0.65,"rankEligible":false},{"stateId":"gemma-4-26b-a4b-it::disabled","configSlug":"gemma-4-26b-a4b-it-no-thinking","modelId":"gemma-4-26b-a4b-it","modelSlug":"gemma-4-26b-a4b-it","name":"Gemma 4 26B A4B IT (disabled)","modelName":"Gemma 4 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":21,"score":141.03,"scoreLo":138.93,"scoreHi":143.17,"reasoningOffset":-1.56,"rankEligible":true},{"stateId":"gemma-4-26b-a4b-it::enabled","configSlug":"gemma-4-26b-a4b-it-thinking","modelId":"gemma-4-26b-a4b-it","modelSlug":"gemma-4-26b-a4b-it","name":"Gemma 4 26B A4B IT (thinking)","modelName":"Gemma 4 26B A4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":35,"score":144.06,"scoreLo":142.14,"scoreHi":146.27,"reasoningOffset":1.48,"rankEligible":true,"rankMean":196.1433,"rankLo":183.0,"rankHi":209.0},{"stateId":"gemma-4-31b-it::default","configSlug":"gemma-4-31b-it","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT","modelName":"Gemma 4 31B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":64,"score":145.88,"scoreLo":143.85,"scoreHi":148.1,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gemma-4-31b-it::effort:high","configSlug":"gemma-4-31b-it-high","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT (high)","modelName":"Gemma 4 31B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":11,"score":147.86,"scoreLo":145.18,"scoreHi":150.77,"reasoningOffset":1.97,"rankEligible":true,"rankMean":155.2289,"rankLo":134.0,"rankHi":176.0},{"stateId":"gemma-4-31b-it::effort:low","configSlug":"gemma-4-31b-it-low","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT (low)","modelName":"Gemma 4 31B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":144.24,"scoreLo":140.04,"scoreHi":148.6,"reasoningOffset":-1.65,"rankEligible":false},{"stateId":"gemma-4-31b-it::effort:max","configSlug":"gemma-4-31b-it-max","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT (max)","modelName":"Gemma 4 31B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":2,"score":146.91,"scoreLo":144.06,"scoreHi":149.99,"reasoningOffset":1.03,"rankEligible":false},{"stateId":"gemma-4-31b-it::effort:medium","configSlug":"gemma-4-31b-it-medium","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT (medium)","modelName":"Gemma 4 31B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":147.06,"scoreLo":143.66,"scoreHi":150.95,"reasoningOffset":1.18,"rankEligible":false},{"stateId":"gemma-4-31b-it::disabled","configSlug":"gemma-4-31b-it-no-thinking","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT (disabled)","modelName":"Gemma 4 31B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":15,"score":143.59,"scoreLo":140.79,"scoreHi":146.6,"reasoningOffset":-2.29,"rankEligible":true},{"stateId":"gemma-4-31b-it::enabled","configSlug":"gemma-4-31b-it-thinking","modelId":"gemma-4-31b-it","modelSlug":"gemma-4-31b-it","name":"Gemma 4 31B IT (thinking)","modelName":"Gemma 4 31B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":46,"score":147.83,"scoreLo":145.68,"scoreHi":150.31,"reasoningOffset":1.94,"rankEligible":true},{"stateId":"gemma-4-e2b-it::default","configSlug":"gemma-4-e2b-it","modelId":"gemma-4-e2b-it","modelSlug":"gemma-4-e2b-it","name":"Gemma 4 E2B IT","modelName":"Gemma 4 E2B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":2,"score":121.13,"scoreLo":116.31,"scoreHi":125.37,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gemma-4-e2b-it::disabled","configSlug":"gemma-4-e2b-it-no-thinking","modelId":"gemma-4-e2b-it","modelSlug":"gemma-4-e2b-it","name":"Gemma 4 E2B IT (disabled)","modelName":"Gemma 4 E2B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":119.63,"scoreLo":114.45,"scoreHi":124.34,"reasoningOffset":-1.5,"rankEligible":true},{"stateId":"gemma-4-e2b-it::enabled","configSlug":"gemma-4-e2b-it-thinking","modelId":"gemma-4-e2b-it","modelSlug":"gemma-4-e2b-it","name":"Gemma 4 E2B IT (thinking)","modelName":"Gemma 4 E2B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":69,"score":120.92,"scoreLo":117.16,"scoreHi":124.11,"reasoningOffset":-0.2,"rankEligible":true,"rankMean":490.3681,"rankLo":457.0,"rankHi":523.0},{"stateId":"gemma-4-e4b-it::default","configSlug":"gemma-4-e4b-it","modelId":"gemma-4-e4b-it","modelSlug":"gemma-4-e4b-it","name":"Gemma 4 E4B IT","modelName":"Gemma 4 E4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":2,"score":132.44,"scoreLo":129.15,"scoreHi":135.55,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gemma-4-e4b-it::disabled","configSlug":"gemma-4-e4b-it-no-thinking","modelId":"gemma-4-e4b-it","modelSlug":"gemma-4-e4b-it","name":"Gemma 4 E4B IT (disabled)","modelName":"Gemma 4 E4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":130.23,"scoreLo":126.27,"scoreHi":133.96,"reasoningOffset":-2.21,"rankEligible":true},{"stateId":"gemma-4-e4b-it::enabled","configSlug":"gemma-4-e4b-it-thinking","modelId":"gemma-4-e4b-it","modelSlug":"gemma-4-e4b-it","name":"Gemma 4 E4B IT (thinking)","modelName":"Gemma 4 E4B IT","organizationSlug":"google","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":29,"score":131.52,"scoreLo":129.18,"scoreHi":133.7,"reasoningOffset":-0.91,"rankEligible":true,"rankMean":354.5639,"rankLo":332.975,"rankHi":380.0},{"stateId":"gemma-7b::default","configSlug":"gemma-7b","modelId":"gemma-7b","modelSlug":"gemma-7b","name":"Gemma 7B","modelName":"Gemma 7B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":42,"score":98.51,"scoreLo":92.4,"scoreHi":103.63,"reasoningOffset":0.0,"rankEligible":true,"rankMean":729.8108,"rankLo":716.975,"rankHi":743.025},{"stateId":"gemma-7b-it::default","configSlug":"gemma-7b-it","modelId":"gemma-7b-it","modelSlug":"gemma-7b-it","name":"Gemma 7B IT","modelName":"Gemma 7B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":89.96,"scoreLo":82.73,"scoreHi":96.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":790.5292,"rankLo":779.0,"rankHi":802.0},{"stateId":"gpt-5-6-luna-pro::effort:extended","configSlug":"genebench-pro-gpt-5-6-luna-pro-genebench-tools","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro (extended)","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":1,"score":162.91,"scoreLo":158.82,"scoreHi":167.76,"reasoningOffset":2.19,"rankEligible":false},{"stateId":"gpt-5-6-terra-pro::effort:extended","configSlug":"genebench-pro-gpt-5-6-terra-pro-genebench-tools","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro (extended)","modelName":"GPT-5.6 Terra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":1,"score":164.28,"scoreLo":159.94,"scoreHi":169.36,"reasoningOffset":1.91,"rankEligible":false},{"stateId":"glm-4-5::default","configSlug":"glm-4-5","modelId":"glm-4-5","modelSlug":"glm-4-5","name":"GLM-4.5","modelName":"GLM-4.5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":25,"score":139.16,"scoreLo":136.6,"scoreHi":141.74,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"glm-4-5-air::default","configSlug":"glm-4-5-air","modelId":"glm-4-5-air","modelSlug":"glm-4-5-air","name":"GLM 4.5 Air","modelName":"GLM 4.5 Air","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":23,"score":140.32,"scoreLo":137.84,"scoreHi":142.96,"reasoningOffset":0.0,"rankEligible":true,"rankMean":235.9606,"rankLo":211.0,"rankHi":266.0},{"stateId":"glm-4-5-air-fp8::default","configSlug":"glm-4-5-air-fp8-helm-capabilities","modelId":"glm-4-5-air-fp8","modelSlug":"glm-4-5-air-fp8","name":"GLM 4.5 Air FP8","modelName":"GLM 4.5 Air FP8","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":5,"score":132.6,"scoreLo":127.24,"scoreHi":137.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":340.2117,"rankLo":276.0,"rankHi":404.025},{"stateId":"glm-4-5-air::disabled","configSlug":"glm-4-5-air-no-thinking","modelId":"glm-4-5-air","modelSlug":"glm-4-5-air","name":"GLM 4.5 Air (disabled)","modelName":"GLM 4.5 Air","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":134.78,"scoreLo":129.52,"scoreHi":140.27,"reasoningOffset":-5.54,"rankEligible":false},{"stateId":"glm-4-5::disabled","configSlug":"glm-4-5-no-thinking","modelId":"glm-4-5","modelSlug":"glm-4-5","name":"GLM-4.5 (disabled)","modelName":"GLM-4.5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":134.5,"scoreLo":129.3,"scoreHi":139.79,"reasoningOffset":-4.66,"rankEligible":false},{"stateId":"glm-4-5::enabled","configSlug":"glm-4-5-thinking","modelId":"glm-4-5","modelSlug":"glm-4-5","name":"GLM-4.5 (thinking)","modelName":"GLM-4.5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":15,"score":140.74,"scoreLo":138.48,"scoreHi":143.09,"reasoningOffset":1.58,"rankEligible":true,"rankMean":230.3597,"rankLo":211.0,"rankHi":254.0},{"stateId":"glm-4-5v::default","configSlug":"glm-4-5v","modelId":"glm-4-5v","modelSlug":"glm-4-5v","name":"GLM-4.5V","modelName":"GLM-4.5V","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":133.83,"scoreLo":130.22,"scoreHi":137.28,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"glm-4-5v::disabled","configSlug":"glm-4-5v-no-thinking","modelId":"glm-4-5v","modelSlug":"glm-4-5v","name":"GLM-4.5V (disabled)","modelName":"GLM-4.5V","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":128.8,"scoreLo":125.05,"scoreHi":132.19,"reasoningOffset":-5.03,"rankEligible":true},{"stateId":"glm-4-5v::enabled","configSlug":"glm-4-5v-thinking","modelId":"glm-4-5v","modelSlug":"glm-4-5v","name":"GLM-4.5V (thinking)","modelName":"GLM-4.5V","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":17,"score":135.39,"scoreLo":132.63,"scoreHi":138.07,"reasoningOffset":1.55,"rankEligible":true,"rankMean":302.8233,"rankLo":271.0,"rankHi":337.0},{"stateId":"glm-4-6::default","configSlug":"glm-4-6","modelId":"glm-4-6","modelSlug":"glm-4-6","name":"GLM 4.6","modelName":"GLM 4.6","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":24,"score":142.13,"scoreLo":139.83,"scoreHi":144.48,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"glm-4-6::effort:minimal","configSlug":"glm-4-6-minimal","modelId":"glm-4-6","modelSlug":"glm-4-6","name":"GLM 4.6 (minimal)","modelName":"GLM 4.6","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":137.58,"scoreLo":132.9,"scoreHi":142.57,"reasoningOffset":-4.55,"rankEligible":false},{"stateId":"glm-4-6::disabled","configSlug":"glm-4-6-no-thinking","modelId":"glm-4-6","modelSlug":"glm-4-6","name":"GLM 4.6 (disabled)","modelName":"GLM 4.6","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":137.42,"scoreLo":134.23,"scoreHi":140.43,"reasoningOffset":-4.71,"rankEligible":true},{"stateId":"glm-4-6::enabled","configSlug":"glm-4-6-thinking","modelId":"glm-4-6","modelSlug":"glm-4-6","name":"GLM 4.6 (thinking)","modelName":"GLM 4.6","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":19,"score":144.5,"scoreLo":142.36,"scoreHi":146.93,"reasoningOffset":2.37,"rankEligible":true,"rankMean":191.4594,"rankLo":176.0,"rankHi":208.0},{"stateId":"glm-4-6v::default","configSlug":"glm-4-6v","modelId":"glm-4-6v","modelSlug":"glm-4-6v","name":"GLM-4.6V","modelName":"GLM-4.6V","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":135.6,"scoreLo":132.26,"scoreHi":138.99,"reasoningOffset":0.0,"rankEligible":true,"rankMean":299.9731,"rankLo":258.0,"rankHi":343.0},{"stateId":"glm-4-6v::disabled","configSlug":"glm-4-6v-no-thinking","modelId":"glm-4-6v","modelSlug":"glm-4-6v","name":"GLM-4.6V (disabled)","modelName":"GLM-4.6V","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":9,"score":129.71,"scoreLo":126.34,"scoreHi":132.86,"reasoningOffset":-5.88,"rankEligible":true},{"stateId":"glm-4-6v::enabled","configSlug":"glm-4-6v-thinking","modelId":"glm-4-6v","modelSlug":"glm-4-6v","name":"GLM-4.6V (thinking)","modelName":"GLM-4.6V","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":133.57,"scoreLo":130.5,"scoreHi":136.6,"reasoningOffset":-2.02,"rankEligible":true},{"stateId":"glm-4-7::default","configSlug":"glm-4-7","modelId":"glm-4-7","modelSlug":"glm-4-7","name":"GLM-4.7","modelName":"GLM-4.7","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":40,"score":146.13,"scoreLo":143.97,"scoreHi":148.55,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"glm-4-7-flash::default","configSlug":"glm-4-7-flash","modelId":"glm-4-7-flash","modelSlug":"glm-4-7-flash","name":"GLM 4.7 Flash","modelName":"GLM 4.7 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":12,"score":136.23,"scoreLo":132.98,"scoreHi":139.37,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"glm-4-7-flash::effort:high","configSlug":"glm-4-7-flash-high","modelId":"glm-4-7-flash","modelSlug":"glm-4-7-flash","name":"GLM 4.7 Flash (high)","modelName":"GLM 4.7 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":135.88,"scoreLo":132.05,"scoreHi":139.82,"reasoningOffset":-0.34,"rankEligible":false},{"stateId":"glm-4-7-flash::disabled","configSlug":"glm-4-7-flash-no-thinking","modelId":"glm-4-7-flash","modelSlug":"glm-4-7-flash","name":"GLM 4.7 Flash (disabled)","modelName":"GLM 4.7 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":9,"score":129.47,"scoreLo":125.83,"scoreHi":132.68,"reasoningOffset":-6.76,"rankEligible":true},{"stateId":"glm-4-7-flash::enabled","configSlug":"glm-4-7-flash-thinking","modelId":"glm-4-7-flash","modelSlug":"glm-4-7-flash","name":"GLM 4.7 Flash (thinking)","modelName":"GLM 4.7 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":138.35,"scoreLo":134.8,"scoreHi":141.94,"reasoningOffset":2.12,"rankEligible":true,"rankMean":262.5311,"rankLo":220.0,"rankHi":308.0},{"stateId":"glm-4-7::effort:high","configSlug":"glm-4-7-high","modelId":"glm-4-7","modelSlug":"glm-4-7","name":"GLM-4.7 (high)","modelName":"GLM-4.7","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":147.04,"scoreLo":144.17,"scoreHi":150.15,"reasoningOffset":0.91,"rankEligible":false},{"stateId":"glm-4-7::effort:minimal","configSlug":"glm-4-7-minimal","modelId":"glm-4-7","modelSlug":"glm-4-7","name":"GLM-4.7 (minimal)","modelName":"GLM-4.7","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":142.14,"scoreLo":137.52,"scoreHi":147.3,"reasoningOffset":-4.0,"rankEligible":false},{"stateId":"glm-4-7::disabled","configSlug":"glm-4-7-no-thinking","modelId":"glm-4-7","modelSlug":"glm-4-7","name":"GLM-4.7 (disabled)","modelName":"GLM-4.7","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":9,"score":140.99,"scoreLo":138.13,"scoreHi":143.98,"reasoningOffset":-5.14,"rankEligible":true},{"stateId":"glm-4-7::enabled","configSlug":"glm-4-7-thinking","modelId":"glm-4-7","modelSlug":"glm-4-7","name":"GLM-4.7 (thinking)","modelName":"GLM-4.7","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":12,"score":148.0,"scoreLo":145.3,"scoreHi":150.92,"reasoningOffset":1.87,"rankEligible":true,"rankMean":153.6408,"rankLo":133.0,"rankHi":174.0},{"stateId":"glm-5::default","configSlug":"glm-5","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5","modelName":"GLM-5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":62,"score":149.45,"scoreLo":147.22,"scoreHi":152.11,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"glm-5-1::default","configSlug":"glm-5-1","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":77,"score":152.84,"scoreLo":150.36,"scoreHi":155.75,"reasoningOffset":0.0,"rankEligible":true,"rankMean":103.4172,"rankLo":96.0,"rankHi":111.0},{"stateId":"glm-5-1::effort:high","configSlug":"glm-5-1-high","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1 (high)","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":6,"score":152.42,"scoreLo":149.44,"scoreHi":155.78,"reasoningOffset":-0.43,"rankEligible":true},{"stateId":"glm-5-1::effort:max","configSlug":"glm-5-1-max","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1 (max)","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":3,"score":153.52,"scoreLo":150.35,"scoreHi":157.11,"reasoningOffset":0.68,"rankEligible":false},{"stateId":"glm-5-1::effort:medium","configSlug":"glm-5-1-medium","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1 (medium)","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":153.11,"scoreLo":149.31,"scoreHi":157.38,"reasoningOffset":0.27,"rankEligible":false},{"stateId":"glm-5-1::effort:minimal","configSlug":"glm-5-1-minimal","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1 (minimal)","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":148.73,"scoreLo":143.99,"scoreHi":154.1,"reasoningOffset":-4.1,"rankEligible":false},{"stateId":"glm-5-1::disabled","configSlug":"glm-5-1-no-thinking","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1 (disabled)","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":146.16,"scoreLo":142.8,"scoreHi":149.74,"reasoningOffset":-6.68,"rankEligible":true},{"stateId":"glm-5-1::enabled","configSlug":"glm-5-1-thinking","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1 (thinking)","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":50,"score":152.71,"scoreLo":150.18,"scoreHi":155.6,"reasoningOffset":-0.13,"rankEligible":true},{"stateId":"glm-5-1::effort:xhigh","configSlug":"glm-5-1-xhigh-genebench-tools","modelId":"glm-5-1","modelSlug":"glm-5-1","name":"GLM-5.1 (xhigh)","modelName":"GLM-5.1","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":153.05,"scoreLo":149.74,"scoreHi":156.83,"reasoningOffset":0.21,"rankEligible":false},{"stateId":"glm-5-2::default","configSlug":"glm-5-2","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":55,"score":156.9,"scoreLo":153.97,"scoreHi":160.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":74.9592,"rankLo":71.0,"rankHi":79.0},{"stateId":"glm-5-2::effort:auto","configSlug":"glm-5-2-auto","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (auto)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"auto","reasoningCondition":"effort:auto","isDefault":false,"n":1,"score":156.58,"scoreLo":151.56,"scoreHi":162.15,"reasoningOffset":-0.32,"rankEligible":false},{"stateId":"glm-5-2::effort:high","configSlug":"glm-5-2-high","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (high)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":20,"score":156.27,"scoreLo":153.25,"scoreHi":159.74,"reasoningOffset":-0.63,"rankEligible":true},{"stateId":"glm-5-2::enabled","configSlug":"glm-5-2-jobbench-opencode","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (enabled)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":157.85,"scoreLo":153.81,"scoreHi":162.32,"reasoningOffset":0.95,"rankEligible":false},{"stateId":"glm-5-2::effort:low","configSlug":"glm-5-2-low","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (low)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":7,"score":154.23,"scoreLo":151.07,"scoreHi":157.97,"reasoningOffset":-2.67,"rankEligible":true},{"stateId":"glm-5-2::effort:max","configSlug":"glm-5-2-max","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (max)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":103,"score":156.43,"scoreLo":153.56,"scoreHi":159.81,"reasoningOffset":-0.47,"rankEligible":true},{"stateId":"glm-5-2::effort:medium","configSlug":"glm-5-2-medium","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (medium)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":7,"score":156.08,"scoreLo":152.6,"scoreHi":160.12,"reasoningOffset":-0.82,"rankEligible":true},{"stateId":"glm-5-2::effort:minimal","configSlug":"glm-5-2-minimal","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (minimal)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":2,"score":152.58,"scoreLo":148.06,"scoreHi":157.64,"reasoningOffset":-4.32,"rankEligible":false},{"stateId":"glm-5-2::disabled","configSlug":"glm-5-2-no-thinking","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (disabled)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":16,"score":148.92,"scoreLo":146.12,"scoreHi":152.07,"reasoningOffset":-7.98,"rankEligible":true},{"stateId":"glm-5-2::effort:xhigh","configSlug":"glm-5-2-xhigh","modelId":"glm-5-2","modelSlug":"glm-5-2","name":"GLM-5.2 (xhigh)","modelName":"GLM-5.2","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":157.44,"scoreLo":153.78,"scoreHi":161.59,"reasoningOffset":0.54,"rankEligible":false},{"stateId":"glm-5-3::default","configSlug":"glm-5-3","modelId":"glm-5-3","modelSlug":"glm-5-3","name":"GLM-5.3","modelName":"GLM-5.3","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":60,"score":160.85,"scoreLo":157.48,"scoreHi":164.83,"reasoningOffset":0.0,"rankEligible":true,"rankMean":39.8369,"rankLo":31.0,"rankHi":48.0},{"stateId":"glm-5-3-flash::default","configSlug":"glm-5-3-flash","modelId":"glm-5-3-flash","modelSlug":"glm-5-3-flash","name":"GLM-5.3 Flash","modelName":"GLM-5.3 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":42,"score":159.24,"scoreLo":156.03,"scoreHi":163.0,"reasoningOffset":0.0,"rankEligible":true,"rankMean":57.12,"rankLo":48.0,"rankHi":66.0},{"stateId":"glm-5-3-flash::effort:high","configSlug":"glm-5-3-flash-high","modelId":"glm-5-3-flash","modelSlug":"glm-5-3-flash","name":"GLM-5.3 Flash (high)","modelName":"GLM-5.3 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":17,"score":157.7,"scoreLo":154.71,"scoreHi":161.31,"reasoningOffset":-1.54,"rankEligible":true},{"stateId":"glm-5-3-flash::effort:low","configSlug":"glm-5-3-flash-low","modelId":"glm-5-3-flash","modelSlug":"glm-5-3-flash","name":"GLM-5.3 Flash (low)","modelName":"GLM-5.3 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":8,"score":154.86,"scoreLo":151.91,"scoreHi":158.31,"reasoningOffset":-4.37,"rankEligible":true},{"stateId":"glm-5-3-flash::effort:max","configSlug":"glm-5-3-flash-max","modelId":"glm-5-3-flash","modelSlug":"glm-5-3-flash","name":"GLM-5.3 Flash (max)","modelName":"GLM-5.3 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":96,"score":158.57,"scoreLo":155.5,"scoreHi":162.21,"reasoningOffset":-0.67,"rankEligible":true},{"stateId":"glm-5-3-flash::enabled","configSlug":"glm-5-3-flash-proximus","modelId":"glm-5-3-flash","modelSlug":"glm-5-3-flash","name":"GLM-5.3 Flash (enabled)","modelName":"GLM-5.3 Flash","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":1,"score":160.08,"scoreLo":156.06,"scoreHi":164.8,"reasoningOffset":0.85,"rankEligible":false},{"stateId":"glm-5-3::effort:high","configSlug":"glm-5-3-high","modelId":"glm-5-3","modelSlug":"glm-5-3","name":"GLM-5.3 (high)","modelName":"GLM-5.3","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":25,"score":160.37,"scoreLo":156.94,"scoreHi":164.41,"reasoningOffset":-0.48,"rankEligible":true},{"stateId":"glm-5-3::effort:low","configSlug":"glm-5-3-low","modelId":"glm-5-3","modelSlug":"glm-5-3","name":"GLM-5.3 (low)","modelName":"GLM-5.3","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":20,"score":158.0,"scoreLo":154.72,"scoreHi":161.85,"reasoningOffset":-2.86,"rankEligible":true},{"stateId":"glm-5-3::effort:max","configSlug":"glm-5-3-max","modelId":"glm-5-3","modelSlug":"glm-5-3","name":"GLM-5.3 (max)","modelName":"GLM-5.3","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":135,"score":160.42,"scoreLo":157.12,"scoreHi":164.24,"reasoningOffset":-0.43,"rankEligible":true},{"stateId":"glm-5-3::enabled","configSlug":"glm-5-3-proximus","modelId":"glm-5-3","modelSlug":"glm-5-3","name":"GLM-5.3 (enabled)","modelName":"GLM-5.3","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":1,"score":162.46,"scoreLo":158.25,"scoreHi":167.62,"reasoningOffset":1.61,"rankEligible":false},{"stateId":"glm-5::effort:auto","configSlug":"glm-5-auto","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5 (auto)","modelName":"GLM-5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"auto","reasoningCondition":"effort:auto","isDefault":false,"n":1,"score":148.97,"scoreLo":143.92,"scoreHi":154.38,"reasoningOffset":-0.48,"rankEligible":false},{"stateId":"glm-5::effort:high","configSlug":"glm-5-high","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5 (high)","modelName":"GLM-5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":149.88,"scoreLo":146.97,"scoreHi":153.1,"reasoningOffset":0.43,"rankEligible":false},{"stateId":"glm-5::effort:minimal","configSlug":"glm-5-minimal","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5 (minimal)","modelName":"GLM-5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":145.29,"scoreLo":140.69,"scoreHi":150.57,"reasoningOffset":-4.16,"rankEligible":false},{"stateId":"glm-5::disabled","configSlug":"glm-5-no-thinking","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5 (disabled)","modelName":"GLM-5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":143.61,"scoreLo":140.14,"scoreHi":147.38,"reasoningOffset":-5.84,"rankEligible":true},{"stateId":"glm-5::enabled","configSlug":"glm-5-thinking","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5 (thinking)","modelName":"GLM-5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":32,"score":150.82,"scoreLo":148.48,"scoreHi":153.56,"reasoningOffset":1.37,"rankEligible":true,"rankMean":122.5764,"rankLo":113.0,"rankHi":133.0},{"stateId":"glm-5-turbo::default","configSlug":"glm-5-turbo","modelId":"glm-5-turbo","modelSlug":"glm-5-turbo","name":"GLM 5 Turbo","modelName":"GLM 5 Turbo","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":147.71,"scoreLo":143.36,"scoreHi":152.4,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"glm-5-turbo::effort:high","configSlug":"glm-5-turbo-high","modelId":"glm-5-turbo","modelSlug":"glm-5-turbo","name":"GLM 5 Turbo (high)","modelName":"GLM 5 Turbo","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":148.27,"scoreLo":143.34,"scoreHi":153.53,"reasoningOffset":0.55,"rankEligible":false},{"stateId":"glm-5-turbo::disabled","configSlug":"glm-5-turbo-no-thinking","modelId":"glm-5-turbo","modelSlug":"glm-5-turbo","name":"GLM 5 Turbo (disabled)","modelName":"GLM 5 Turbo","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":141.65,"scoreLo":135.84,"scoreHi":148.13,"reasoningOffset":-6.07,"rankEligible":false},{"stateId":"glm-5-turbo::enabled","configSlug":"glm-5-turbo-thinking","modelId":"glm-5-turbo","modelSlug":"glm-5-turbo","name":"GLM 5 Turbo (thinking)","modelName":"GLM 5 Turbo","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":6,"score":147.98,"scoreLo":144.37,"scoreHi":151.72,"reasoningOffset":0.27,"rankEligible":true,"rankMean":153.7683,"rankLo":121.0,"rankHi":188.0},{"stateId":"glm-5::effort:xhigh","configSlug":"glm-5-xhigh","modelId":"glm-5","modelSlug":"glm-5","name":"GLM-5 (xhigh)","modelName":"GLM-5","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":149.94,"scoreLo":146.73,"scoreHi":153.58,"reasoningOffset":0.49,"rankEligible":false},{"stateId":"glm-5v-turbo::default","configSlug":"glm-5v-turbo","modelId":"glm-5v-turbo","modelSlug":"glm-5v-turbo","name":"GLM 5V Turbo","modelName":"GLM 5V Turbo","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":148.7,"scoreLo":146.02,"scoreHi":151.72,"reasoningOffset":0.0,"rankEligible":true,"rankMean":145.9911,"rankLo":126.0,"rankHi":167.0},{"stateId":"glm-5v-turbo::effort:low","configSlug":"glm-5v-turbo-low","modelId":"glm-5v-turbo","modelSlug":"glm-5v-turbo","name":"GLM 5V Turbo (low)","modelName":"GLM 5V Turbo","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":146.23,"scoreLo":141.87,"scoreHi":150.8,"reasoningOffset":-2.46,"rankEligible":false},{"stateId":"glm-5v-turbo::effort:minimal","configSlug":"glm-5v-turbo-minimal","modelId":"glm-5v-turbo","modelSlug":"glm-5v-turbo","name":"GLM 5V Turbo (minimal)","modelName":"GLM 5V Turbo","organizationSlug":"zhipu-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":144.32,"scoreLo":139.46,"scoreHi":149.42,"reasoningOffset":-4.38,"rankEligible":false},{"stateId":"gopher-280b::default","configSlug":"gopher-280b","modelId":"gopher-280b","modelSlug":"gopher-280b","name":"Gopher (280B)","modelName":"Gopher (280B)","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":95.56,"scoreLo":86.71,"scoreHi":103.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":752.8564,"rankLo":705.975,"rankHi":794.0},{"stateId":"gpt-2-124m::default","configSlug":"gpt-2-124m","modelId":"gpt-2-124m","modelSlug":"gpt-2-124m","name":"GPT-2 (124M)","modelName":"GPT-2 (124M)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":33.54,"scoreLo":13.25,"scoreHi":50.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":884.7172,"rankLo":883.0,"rankHi":886.0},{"stateId":"gpt-2-1558m::default","configSlug":"gpt-2-1558m","modelId":"gpt-2-1558m","modelSlug":"gpt-2-1558m","name":"GPT-2 (1.5B)","modelName":"GPT-2 (1.5B)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":53.01,"scoreLo":37.34,"scoreHi":66.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":878.0106,"rankLo":872.0,"rankHi":883.0},{"stateId":"gpt-2-355m::default","configSlug":"gpt-2-355m","modelId":"gpt-2-355m","modelSlug":"gpt-2-355m","name":"GPT-2 (355M)","modelName":"GPT-2 (355M)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":44.67,"scoreLo":23.48,"scoreHi":60.8,"reasoningOffset":0.0,"rankEligible":true,"rankMean":882.0236,"rankLo":876.0,"rankHi":885.0},{"stateId":"gpt-2-774m::default","configSlug":"gpt-2-774m","modelId":"gpt-2-774m","modelSlug":"gpt-2-774m","name":"GPT-2 (774M)","modelName":"GPT-2 (774M)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":49.22,"scoreLo":31.54,"scoreHi":63.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":880.1822,"rankLo":873.0,"rankHi":884.0},{"stateId":"gpt-3-13b::default","configSlug":"gpt-3-13b","modelId":"gpt-3-13b","modelSlug":"gpt-3-13b","name":"GPT-3 13B","modelName":"GPT-3 13B","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":80.48,"scoreLo":68.14,"scoreHi":90.82,"reasoningOffset":0.0,"rankEligible":true,"rankMean":830.6028,"rankLo":799.0,"rankHi":856.0},{"stateId":"gpt-3-175b::default","configSlug":"gpt-3-175b","modelId":"gpt-3-175b","modelSlug":"gpt-3-175b","name":"GPT-3 175B","modelName":"GPT-3 175B","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":88.06,"scoreLo":76.64,"scoreHi":98.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":798.1139,"rankLo":745.0,"rankHi":834.0},{"stateId":"gpt-3-2-7b::default","configSlug":"gpt-3-2-7b","modelId":"gpt-3-2-7b","modelSlug":"gpt-3-2-7b","name":"GPT-3 2.7B","modelName":"GPT-3 2.7B","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":69.68,"scoreLo":55.83,"scoreHi":81.44,"reasoningOffset":0.0,"rankEligible":true,"rankMean":859.4628,"rankLo":841.0,"rankHi":871.0},{"stateId":"gpt-3-5-turbo-0125::default","configSlug":"gpt-3-5-turbo-0125","modelId":"gpt-3-5-turbo-0125","modelSlug":"gpt-3-5-turbo-0125","name":"GPT-3.5 Turbo 0125","modelName":"GPT-3.5 Turbo 0125","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":109.13,"scoreLo":103.54,"scoreHi":113.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":633.0453,"rankLo":595.0,"rankHi":670.0},{"stateId":"gpt-3-5-turbo-0301::default","configSlug":"gpt-3-5-turbo-0301","modelId":"gpt-3-5-turbo-0301","modelSlug":"gpt-3-5-turbo-0301","name":"GPT-3.5 Turbo 0301","modelName":"GPT-3.5 Turbo 0301","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":102.44,"scoreLo":93.71,"scoreHi":109.55,"reasoningOffset":0.0,"rankEligible":true,"rankMean":697.8956,"rankLo":641.0,"rankHi":754.025},{"stateId":"gpt-3-5-turbo-0613::default","configSlug":"gpt-3-5-turbo-0613","modelId":"gpt-3-5-turbo-0613","modelSlug":"gpt-3-5-turbo-0613","name":"GPT-3.5 Turbo 0613","modelName":"GPT-3.5 Turbo 0613","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":108.05,"scoreLo":101.36,"scoreHi":114.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":644.5044,"rankLo":589.0,"rankHi":694.0},{"stateId":"gpt-3-5-turbo-1106::default","configSlug":"gpt-3-5-turbo-1106","modelId":"gpt-3-5-turbo-1106","modelSlug":"gpt-3-5-turbo-1106","name":"GPT-3.5 Turbo 1106","modelName":"GPT-3.5 Turbo 1106","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":105.66,"scoreLo":99.37,"scoreHi":111.14,"reasoningOffset":0.0,"rankEligible":true,"rankMean":669.7022,"rankLo":630.0,"rankHi":707.0},{"stateId":"gpt-3-6-7b::default","configSlug":"gpt-3-6-7b","modelId":"gpt-3-6-7b","modelSlug":"gpt-3-6-7b","name":"GPT-3 6.7B","modelName":"GPT-3 6.7B","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":75.2,"scoreLo":63.27,"scoreHi":85.89,"reasoningOffset":0.0,"rankEligible":true,"rankMean":847.3431,"rankLo":821.0,"rankHi":865.0},{"stateId":"gpt-3-large::default","configSlug":"gpt-3-large","modelId":"gpt-3-large","modelSlug":"gpt-3-large","name":"GPT-3 Large","modelName":"GPT-3 Large","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":55.42,"scoreLo":39.97,"scoreHi":69.12,"reasoningOffset":0.0,"rankEligible":true,"rankMean":876.1547,"rankLo":868.0,"rankHi":882.0},{"stateId":"gpt-3-medium::default","configSlug":"gpt-3-medium","modelId":"gpt-3-medium","modelSlug":"gpt-3-medium","name":"GPT-3 Medium","modelName":"GPT-3 Medium","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":45.39,"scoreLo":28.77,"scoreHi":59.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":882.0978,"rankLo":878.0,"rankHi":885.0},{"stateId":"gpt-3-small::default","configSlug":"gpt-3-small","modelId":"gpt-3-small","modelSlug":"gpt-3-small","name":"GPT-3 Small","modelName":"GPT-3 Small","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":18.7,"scoreLo":-3.79,"scoreHi":36.35,"reasoningOffset":0.0,"rankEligible":true,"rankMean":885.9711,"rankLo":885.0,"rankHi":886.0},{"stateId":"gpt-3-xl::default","configSlug":"gpt-3-xl","modelId":"gpt-3-xl","modelSlug":"gpt-3-xl","name":"GPT-3 XL","modelName":"GPT-3 XL","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":61.16,"scoreLo":45.97,"scoreHi":73.63,"reasoningOffset":0.0,"rankEligible":true,"rankMean":870.8692,"rankLo":862.0,"rankHi":879.0},{"stateId":"gpt-4-0125-preview::default","configSlug":"gpt-4-0125-preview","modelId":"gpt-4-0125-preview","modelSlug":"gpt-4-0125-preview","name":"GPT-4 0125 Preview","modelName":"GPT-4 0125 Preview","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":124.85,"scoreLo":121.68,"scoreHi":127.69,"reasoningOffset":0.0,"rankEligible":true,"rankMean":437.4772,"rankLo":410.0,"rankHi":468.0},{"stateId":"gpt-4-0314::default","configSlug":"gpt-4-0314","modelId":"gpt-4-0314","modelSlug":"gpt-4-0314","name":"GPT-4 (0314)","modelName":"GPT-4 (0314)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":119.57,"scoreLo":115.48,"scoreHi":123.16,"reasoningOffset":0.0,"rankEligible":true,"rankMean":508.1242,"rankLo":470.0,"rankHi":542.0},{"stateId":"gpt-4-0613::default","configSlug":"gpt-4-0613","modelId":"gpt-4-0613","modelSlug":"gpt-4-0613","name":"GPT-4 0613","modelName":"GPT-4 0613","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":121.44,"scoreLo":117.28,"scoreHi":125.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":483.0714,"rankLo":441.0,"rankHi":525.0},{"stateId":"gpt-4-1::default","configSlug":"gpt-4-1","modelId":"gpt-4-1","modelSlug":"gpt-4-1","name":"GPT-4.1","modelName":"GPT-4.1","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":131,"score":137.58,"scoreLo":135.97,"scoreHi":139.18,"reasoningOffset":0.0,"rankEligible":true,"rankMean":272.9767,"rankLo":258.0,"rankHi":288.0},{"stateId":"gpt-4-1-mini::default","configSlug":"gpt-4-1-mini","modelId":"gpt-4-1-mini","modelSlug":"gpt-4-1-mini","name":"GPT-4.1 Mini","modelName":"GPT-4.1 Mini","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":88,"score":133.45,"scoreLo":131.56,"scoreHi":135.24,"reasoningOffset":0.0,"rankEligible":true,"rankMean":329.6925,"rankLo":310.0,"rankHi":348.0},{"stateId":"gpt-4-1-nano::default","configSlug":"gpt-4-1-nano","modelId":"gpt-4-1-nano","modelSlug":"gpt-4-1-nano","name":"GPT-4.1 Nano","modelName":"GPT-4.1 Nano","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":81,"score":119.63,"scoreLo":116.42,"scoreHi":122.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":508.27,"rankLo":488.0,"rankHi":527.0},{"stateId":"gpt-4-1106-preview::default","configSlug":"gpt-4-1106-preview","modelId":"gpt-4-1106-preview","modelSlug":"gpt-4-1106-preview","name":"GPT-4 Turbo (1106 Preview)","modelName":"GPT-4 Turbo (1106 Preview)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":124.63,"scoreLo":121.25,"scoreHi":127.65,"reasoningOffset":0.0,"rankEligible":true,"rankMean":440.2672,"rankLo":410.0,"rankHi":473.0},{"stateId":"gpt-4-5-preview::default","configSlug":"gpt-4-5-preview","modelId":"gpt-4-5-preview","modelSlug":"gpt-4-5-preview","name":"GPT-4.5","modelName":"GPT-4.5","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":51,"score":138.28,"scoreLo":136.24,"scoreHi":140.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":263.3442,"rankLo":242.0,"rankHi":286.0},{"stateId":"gpt-4-turbo-2024-04-09::default","configSlug":"gpt-4-turbo-2024-04-09","modelId":"gpt-4-turbo-2024-04-09","modelSlug":"gpt-4-turbo-2024-04-09","name":"GPT-4 Turbo","modelName":"GPT-4 Turbo","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":48,"score":126.48,"scoreLo":123.96,"scoreHi":128.87,"reasoningOffset":0.0,"rankEligible":true,"rankMean":417.5336,"rankLo":397.0,"rankHi":440.0},{"stateId":"gpt-4-vision-preview::default","configSlug":"gpt-4-vision-preview","modelId":"gpt-4-vision-preview","modelSlug":"gpt-4-vision-preview","name":"GPT-4 Turbo with Vision","modelName":"GPT-4 Turbo with Vision","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":114.82,"scoreLo":107.96,"scoreHi":120.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":564.6492,"rankLo":502.0,"rankHi":637.0},{"stateId":"gpt-4o-2024-05-13::default","configSlug":"gpt-4o-2024-05-13","modelId":"gpt-4o-2024-05-13","modelSlug":"gpt-4o-2024-05-13","name":"GPT-4o (2024-05-13)","modelName":"GPT-4o (2024-05-13)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":59,"score":129.59,"scoreLo":127.33,"scoreHi":131.72,"reasoningOffset":0.0,"rankEligible":true,"rankMean":380.6486,"rankLo":357.0,"rankHi":402.0},{"stateId":"gpt-4o-2024-08-06::default","configSlug":"gpt-4o-2024-08-06","modelId":"gpt-4o-2024-08-06","modelSlug":"gpt-4o-2024-08-06","name":"GPT-4o (2024-08-06)","modelName":"GPT-4o (2024-08-06)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":73,"score":129.58,"scoreLo":127.38,"scoreHi":131.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":380.8136,"rankLo":359.0,"rankHi":401.0},{"stateId":"gpt-4o-2024-11-20::default","configSlug":"gpt-4o-2024-11-20","modelId":"gpt-4o-2024-11-20","modelSlug":"gpt-4o-2024-11-20","name":"GPT-4o (2024-11-20)","modelName":"GPT-4o (2024-11-20)","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":74,"score":129.81,"scoreLo":127.58,"scoreHi":131.86,"reasoningOffset":0.0,"rankEligible":true,"rankMean":377.705,"rankLo":357.0,"rankHi":399.025},{"stateId":"gpt-4o-mini::default","configSlug":"gpt-4o-mini","modelId":"gpt-4o-mini","modelSlug":"gpt-4o-mini","name":"GPT-4o Mini","modelName":"GPT-4o Mini","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":106,"score":121.49,"scoreLo":118.71,"scoreHi":123.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":482.5356,"rankLo":462.0,"rankHi":503.0},{"stateId":"gpt-5::default","configSlug":"gpt-5","modelId":"gpt-5","modelSlug":"gpt-5","name":"GPT-5","modelName":"GPT-5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":64,"score":149.32,"scoreLo":147.27,"scoreHi":151.69,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-1::default","configSlug":"gpt-5-1","modelId":"gpt-5-1","modelSlug":"gpt-5-1","name":"GPT-5.1","modelName":"GPT-5.1","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":31,"score":148.97,"scoreLo":146.73,"scoreHi":151.57,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-1-chat-latest::default","configSlug":"gpt-5-1-chat-latest","modelId":"gpt-5-1-chat-latest","modelSlug":"gpt-5-1-chat-latest","name":"GPT-5.1 Instant","modelName":"GPT-5.1 Instant","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":139.3,"scoreLo":135.62,"scoreHi":142.91,"reasoningOffset":2.4,"rankEligible":true,"rankMean":249.8097,"rankLo":210.0,"rankHi":298.0},{"stateId":"gpt-5-1-chat-latest::effort:high","configSlug":"gpt-5-1-chat-latest-high","modelId":"gpt-5-1-chat-latest","modelSlug":"gpt-5-1-chat-latest","name":"GPT-5.1 Instant (high)","modelName":"GPT-5.1 Instant","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":137.43,"scoreLo":132.92,"scoreHi":141.8,"reasoningOffset":0.53,"rankEligible":false},{"stateId":"gpt-5-1-chat-latest::effort:low","configSlug":"gpt-5-1-chat-latest-low","modelId":"gpt-5-1-chat-latest","modelSlug":"gpt-5-1-chat-latest","name":"GPT-5.1 Instant (low)","modelName":"GPT-5.1 Instant","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":135.12,"scoreLo":130.1,"scoreHi":140.42,"reasoningOffset":-1.78,"rankEligible":false},{"stateId":"gpt-5-1-chat-latest::effort:medium","configSlug":"gpt-5-1-chat-latest-medium","modelId":"gpt-5-1-chat-latest","modelSlug":"gpt-5-1-chat-latest","name":"GPT-5.1 Instant (medium)","modelName":"GPT-5.1 Instant","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":136.98,"scoreLo":132.25,"scoreHi":141.79,"reasoningOffset":0.08,"rankEligible":false},{"stateId":"gpt-5-1-codex::default","configSlug":"gpt-5-1-codex","modelId":"gpt-5-1-codex","modelSlug":"gpt-5-1-codex","name":"GPT-5.1-Codex","modelName":"GPT-5.1-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":7,"score":147.66,"scoreLo":145.04,"scoreHi":150.52,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-1-codex::effort:high","configSlug":"gpt-5-1-codex-high","modelId":"gpt-5-1-codex","modelSlug":"gpt-5-1-codex","name":"GPT-5.1-Codex (high)","modelName":"GPT-5.1-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":16,"score":149.64,"scoreLo":146.96,"scoreHi":152.66,"reasoningOffset":1.98,"rankEligible":true,"rankMean":135.7006,"rankLo":116.0,"rankHi":155.0},{"stateId":"gpt-5-1-codex::effort:low","configSlug":"gpt-5-1-codex-low","modelId":"gpt-5-1-codex","modelSlug":"gpt-5-1-codex","name":"GPT-5.1-Codex (low)","modelName":"GPT-5.1-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":145.97,"scoreLo":141.95,"scoreHi":150.34,"reasoningOffset":-1.7,"rankEligible":false},{"stateId":"gpt-5-1-codex-max::effort:xhigh","configSlug":"gpt-5-1-codex-max","modelId":"gpt-5-1-codex-max","modelSlug":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max (xhigh)","modelName":"GPT-5.1-Codex-Max","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":true,"n":6,"score":152.22,"scoreLo":148.8,"scoreHi":156.17,"reasoningOffset":1.51,"rankEligible":true,"rankMean":109.3308,"rankLo":85.0,"rankHi":136.0},{"stateId":"gpt-5-1-codex-max::effort:high","configSlug":"gpt-5-1-codex-max-high","modelId":"gpt-5-1-codex-max","modelSlug":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max (high)","modelName":"GPT-5.1-Codex-Max","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":6,"score":151.03,"scoreLo":147.8,"scoreHi":154.62,"reasoningOffset":0.32,"rankEligible":true},{"stateId":"gpt-5-1-codex-max::effort:low","configSlug":"gpt-5-1-codex-max-low","modelId":"gpt-5-1-codex-max","modelSlug":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max (low)","modelName":"GPT-5.1-Codex-Max","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":148.43,"scoreLo":143.85,"scoreHi":153.38,"reasoningOffset":-2.28,"rankEligible":false},{"stateId":"gpt-5-1-codex-max::effort:medium","configSlug":"gpt-5-1-codex-max-medium","modelId":"gpt-5-1-codex-max","modelSlug":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max (medium)","modelName":"GPT-5.1-Codex-Max","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":150.26,"scoreLo":145.93,"scoreHi":155.18,"reasoningOffset":-0.45,"rankEligible":false},{"stateId":"gpt-5-1-codex-max::default","configSlug":"gpt-5-1-codex-max-unknown","modelId":"gpt-5-1-codex-max","modelSlug":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max","modelName":"GPT-5.1-Codex-Max","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":3,"score":150.71,"scoreLo":147.81,"scoreHi":154.01,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gpt-5-1-codex::effort:medium","configSlug":"gpt-5-1-codex-medium","modelId":"gpt-5-1-codex","modelSlug":"gpt-5-1-codex","name":"GPT-5.1-Codex (medium)","modelName":"GPT-5.1-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":147.38,"scoreLo":142.86,"scoreHi":152.07,"reasoningOffset":-0.28,"rankEligible":false},{"stateId":"gpt-5-1-codex-mini::default","configSlug":"gpt-5-1-codex-mini","modelId":"gpt-5-1-codex-mini","modelSlug":"gpt-5-1-codex-mini","name":"GPT-5.1-Codex mini","modelName":"GPT-5.1-Codex mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":6,"score":146.45,"scoreLo":143.37,"scoreHi":149.64,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-1-codex-mini::effort:high","configSlug":"gpt-5-1-codex-mini-high","modelId":"gpt-5-1-codex-mini","modelSlug":"gpt-5-1-codex-mini","name":"GPT-5.1-Codex mini (high)","modelName":"GPT-5.1-Codex mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":11,"score":147.09,"scoreLo":144.21,"scoreHi":150.27,"reasoningOffset":0.64,"rankEligible":true,"rankMean":163.5303,"rankLo":138.0,"rankHi":188.0},{"stateId":"gpt-5-1-codex-mini::effort:low","configSlug":"gpt-5-1-codex-mini-low","modelId":"gpt-5-1-codex-mini","modelSlug":"gpt-5-1-codex-mini","name":"GPT-5.1-Codex mini (low)","modelName":"GPT-5.1-Codex mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":144.75,"scoreLo":140.08,"scoreHi":149.53,"reasoningOffset":-1.7,"rankEligible":false},{"stateId":"gpt-5-1-codex-mini::effort:medium","configSlug":"gpt-5-1-codex-mini-medium","modelId":"gpt-5-1-codex-mini","modelSlug":"gpt-5-1-codex-mini","name":"GPT-5.1-Codex mini (medium)","modelName":"GPT-5.1-Codex mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":146.49,"scoreLo":141.9,"scoreHi":151.23,"reasoningOffset":0.04,"rankEligible":false},{"stateId":"gpt-5-1::effort:high","configSlug":"gpt-5-1-high","modelId":"gpt-5-1","modelSlug":"gpt-5-1","name":"GPT-5.1 (high)","modelName":"GPT-5.1","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":89,"score":150.43,"scoreLo":148.15,"scoreHi":153.05,"reasoningOffset":1.46,"rankEligible":true,"rankMean":126.9311,"rankLo":118.0,"rankHi":136.0},{"stateId":"gpt-5-1::effort:low","configSlug":"gpt-5-1-low","modelId":"gpt-5-1","modelSlug":"gpt-5-1","name":"GPT-5.1 (low)","modelName":"GPT-5.1","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":11,"score":146.66,"scoreLo":144.44,"scoreHi":149.1,"reasoningOffset":-2.31,"rankEligible":true},{"stateId":"gpt-5-1::effort:medium","configSlug":"gpt-5-1-medium","modelId":"gpt-5-1","modelSlug":"gpt-5-1","name":"GPT-5.1 (medium)","modelName":"GPT-5.1","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":19,"score":149.24,"scoreLo":146.91,"scoreHi":151.88,"reasoningOffset":0.27,"rankEligible":true},{"stateId":"gpt-5-1::disabled","configSlug":"gpt-5-1-no-thinking","modelId":"gpt-5-1","modelSlug":"gpt-5-1","name":"GPT-5.1 (disabled)","modelName":"GPT-5.1","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":23,"score":141.32,"scoreLo":139.19,"scoreHi":143.52,"reasoningOffset":-7.65,"rankEligible":true},{"stateId":"gpt-5-1::enabled","configSlug":"gpt-5-1-thinking","modelId":"gpt-5-1","modelSlug":"gpt-5-1","name":"GPT-5.1 (thinking)","modelName":"GPT-5.1","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":149.92,"scoreLo":147.01,"scoreHi":153.16,"reasoningOffset":0.95,"rankEligible":true},{"stateId":"gpt-5-2::default","configSlug":"gpt-5-2","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":98,"score":152.47,"scoreLo":149.99,"scoreHi":155.37,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-2-codex::default","configSlug":"gpt-5-2-codex","modelId":"gpt-5-2-codex","modelSlug":"gpt-5-2-codex","name":"GPT-5.2-Codex","modelName":"GPT-5.2-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":13,"score":151.93,"scoreLo":149.26,"scoreHi":155.1,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-2-codex::effort:high","configSlug":"gpt-5-2-codex-high","modelId":"gpt-5-2-codex","modelSlug":"gpt-5-2-codex","name":"GPT-5.2-Codex (high)","modelName":"GPT-5.2-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":8,"score":152.44,"scoreLo":149.4,"scoreHi":155.82,"reasoningOffset":0.52,"rankEligible":true,"rankMean":107.0306,"rankLo":89.0,"rankHi":128.0},{"stateId":"gpt-5-2-codex::effort:low","configSlug":"gpt-5-2-codex-low","modelId":"gpt-5-2-codex","modelSlug":"gpt-5-2-codex","name":"GPT-5.2-Codex (low)","modelName":"GPT-5.2-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":149.71,"scoreLo":145.06,"scoreHi":154.59,"reasoningOffset":-2.22,"rankEligible":false},{"stateId":"gpt-5-2-codex::effort:medium","configSlug":"gpt-5-2-codex-medium","modelId":"gpt-5-2-codex","modelSlug":"gpt-5-2-codex","name":"GPT-5.2-Codex (medium)","modelName":"GPT-5.2-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":151.56,"scoreLo":147.22,"scoreHi":156.35,"reasoningOffset":-0.36,"rankEligible":false},{"stateId":"gpt-5-2-codex::effort:xhigh","configSlug":"gpt-5-2-codex-xhigh","modelId":"gpt-5-2-codex","modelSlug":"gpt-5-2-codex","name":"GPT-5.2-Codex (xhigh)","modelName":"GPT-5.2-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":13,"score":152.42,"scoreLo":149.4,"scoreHi":155.81,"reasoningOffset":0.49,"rankEligible":true},{"stateId":"gpt-5-2::effort:extended","configSlug":"gpt-5-2-extended-chatgpt-plus","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (extended)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":1,"score":154.54,"scoreLo":150.85,"scoreHi":158.55,"reasoningOffset":2.07,"rankEligible":false},{"stateId":"gpt-5-2::effort:heavy","configSlug":"gpt-5-2-heavy","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (heavy)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"heavy","reasoningCondition":"effort:heavy","isDefault":false,"n":1,"score":155.0,"scoreLo":150.34,"scoreHi":159.85,"reasoningOffset":2.53,"rankEligible":false},{"stateId":"gpt-5-2::effort:high","configSlug":"gpt-5-2-high","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (high)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":60,"score":153.69,"scoreLo":151.13,"scoreHi":156.67,"reasoningOffset":1.22,"rankEligible":true},{"stateId":"gpt-5-2::effort:low","configSlug":"gpt-5-2-low","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (low)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":20,"score":150.56,"scoreLo":148.1,"scoreHi":153.32,"reasoningOffset":-1.91,"rankEligible":true},{"stateId":"gpt-5-2::effort:medium","configSlug":"gpt-5-2-medium","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (medium)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":37,"score":153.05,"scoreLo":150.46,"scoreHi":155.95,"reasoningOffset":0.58,"rankEligible":true},{"stateId":"gpt-5-2::effort:minimal","configSlug":"gpt-5-2-minimal","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (minimal)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":147.93,"scoreLo":142.97,"scoreHi":152.92,"reasoningOffset":-4.54,"rankEligible":false},{"stateId":"gpt-5-2::disabled","configSlug":"gpt-5-2-no-thinking","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (disabled)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":32,"score":145.56,"scoreLo":143.43,"scoreHi":147.93,"reasoningOffset":-6.92,"rankEligible":true},{"stateId":"gpt-5-2-pro::default","configSlug":"gpt-5-2-pro","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro","modelName":"GPT-5.2 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":7,"score":155.71,"scoreLo":152.58,"scoreHi":159.33,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-2-pro::effort:extended","configSlug":"gpt-5-2-pro-extended-genebench-pro-genebench-tools","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro (extended)","modelName":"GPT-5.2 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":1,"score":157.8,"scoreLo":153.98,"scoreHi":162.25,"reasoningOffset":2.09,"rankEligible":false},{"stateId":"gpt-5-2-pro::effort:high","configSlug":"gpt-5-2-pro-high","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro (high)","modelName":"GPT-5.2 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":156.58,"scoreLo":153.36,"scoreHi":160.28,"reasoningOffset":0.87,"rankEligible":false},{"stateId":"gpt-5-2-pro::effort:medium","configSlug":"gpt-5-2-pro-medium","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro (medium)","modelName":"GPT-5.2 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":155.53,"scoreLo":152.5,"scoreHi":159.26,"reasoningOffset":-0.18,"rankEligible":false},{"stateId":"gpt-5-2-pro::effort:minimal","configSlug":"gpt-5-2-pro-minimal","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro (minimal)","modelName":"GPT-5.2 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":151.57,"scoreLo":146.71,"scoreHi":157.08,"reasoningOffset":-4.14,"rankEligible":false},{"stateId":"gpt-5-2-pro::enabled","configSlug":"gpt-5-2-pro-taxbench","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro (enabled)","modelName":"GPT-5.2 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":156.69,"scoreLo":152.4,"scoreHi":161.42,"reasoningOffset":0.98,"rankEligible":false},{"stateId":"gpt-5-2-pro::effort:xhigh","configSlug":"gpt-5-2-pro-xhigh","modelId":"gpt-5-2-pro","modelSlug":"gpt-5-2-pro","name":"GPT-5.2 Pro (xhigh)","modelName":"GPT-5.2 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":14,"score":157.7,"scoreLo":154.51,"scoreHi":161.46,"reasoningOffset":1.99,"rankEligible":true,"rankMean":70.0506,"rankLo":58.0,"rankHi":78.0},{"stateId":"gpt-5-2::enabled","configSlug":"gpt-5-2-thinking","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (thinking)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":14,"score":153.94,"scoreLo":150.79,"scoreHi":157.75,"reasoningOffset":1.47,"rankEligible":true},{"stateId":"gpt-5-2::effort:xhigh","configSlug":"gpt-5-2-xhigh","modelId":"gpt-5-2","modelSlug":"gpt-5-2","name":"GPT-5.2 (xhigh)","modelName":"GPT-5.2","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":87,"score":155.23,"scoreLo":152.5,"scoreHi":158.47,"reasoningOffset":2.76,"rankEligible":true,"rankMean":83.2731,"rankLo":79.0,"rankHi":89.0},{"stateId":"gpt-5-3-codex::default","configSlug":"gpt-5-3-codex","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":155.04,"scoreLo":152.11,"scoreHi":158.41,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-3-codex::enabled","configSlug":"gpt-5-3-codex-cli","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex (enabled)","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":157.03,"scoreLo":152.87,"scoreHi":161.9,"reasoningOffset":1.99,"rankEligible":false},{"stateId":"gpt-5-3-codex::effort:high","configSlug":"gpt-5-3-codex-high","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex (high)","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":10,"score":156.27,"scoreLo":153.1,"scoreHi":159.91,"reasoningOffset":1.23,"rankEligible":true},{"stateId":"gpt-5-3-codex::effort:low","configSlug":"gpt-5-3-codex-low","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex (low)","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":152.93,"scoreLo":148.82,"scoreHi":157.49,"reasoningOffset":-2.11,"rankEligible":false},{"stateId":"gpt-5-3-codex::effort:medium","configSlug":"gpt-5-3-codex-medium","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex (medium)","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":3,"score":155.04,"scoreLo":151.01,"scoreHi":159.6,"reasoningOffset":-0.0,"rankEligible":false},{"stateId":"gpt-5-3-codex::effort:minimal","configSlug":"gpt-5-3-codex-minimal","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex (minimal)","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":150.9,"scoreLo":146.23,"scoreHi":156.29,"reasoningOffset":-4.14,"rankEligible":false},{"stateId":"gpt-5-3-codex::disabled","configSlug":"gpt-5-3-codex-no-thinking","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex (disabled)","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":148.69,"scoreLo":142.05,"scoreHi":155.63,"reasoningOffset":-6.35,"rankEligible":false},{"stateId":"gpt-5-3-codex-spark::default","configSlug":"gpt-5-3-codex-spark","modelId":"gpt-5-3-codex-spark","modelSlug":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark","modelName":"GPT-5.3-Codex-Spark","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":150.04,"scoreLo":146.65,"scoreHi":153.73,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gpt-5-3-codex-spark::effort:high","configSlug":"gpt-5-3-codex-spark-high","modelId":"gpt-5-3-codex-spark","modelSlug":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark (high)","modelName":"GPT-5.3-Codex-Spark","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":149.97,"scoreLo":146.17,"scoreHi":154.19,"reasoningOffset":-0.07,"rankEligible":false},{"stateId":"gpt-5-3-codex-spark::effort:low","configSlug":"gpt-5-3-codex-spark-low","modelId":"gpt-5-3-codex-spark","modelSlug":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark (low)","modelName":"GPT-5.3-Codex-Spark","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":147.88,"scoreLo":143.22,"scoreHi":152.81,"reasoningOffset":-2.16,"rankEligible":false},{"stateId":"gpt-5-3-codex-spark::effort:medium","configSlug":"gpt-5-3-codex-spark-medium","modelId":"gpt-5-3-codex-spark","modelSlug":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark (medium)","modelName":"GPT-5.3-Codex-Spark","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":149.77,"scoreLo":145.08,"scoreHi":154.68,"reasoningOffset":-0.27,"rankEligible":false},{"stateId":"gpt-5-3-codex-spark::effort:xhigh","configSlug":"gpt-5-3-codex-spark-xhigh","modelId":"gpt-5-3-codex-spark","modelSlug":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark (xhigh)","modelName":"GPT-5.3-Codex-Spark","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":150.87,"scoreLo":146.77,"scoreHi":155.35,"reasoningOffset":0.82,"rankEligible":false},{"stateId":"gpt-5-3-codex::effort:xhigh","configSlug":"gpt-5-3-codex-xhigh","modelId":"gpt-5-3-codex","modelSlug":"gpt-5-3-codex","name":"GPT-5.3-Codex (xhigh)","modelName":"GPT-5.3-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":26,"score":157.31,"scoreLo":154.19,"scoreHi":161.07,"reasoningOffset":2.27,"rankEligible":true,"rankMean":72.5197,"rankLo":64.0,"rankHi":79.0},{"stateId":"gpt-5-4::default","configSlug":"gpt-5-4","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":57,"score":156.29,"scoreLo":153.48,"scoreHi":159.64,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-4::effort:extended","configSlug":"gpt-5-4-extended-chatgpt-plus","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (extended)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":1,"score":158.26,"scoreLo":154.31,"scoreHi":162.85,"reasoningOffset":1.96,"rankEligible":false},{"stateId":"gpt-5-4::effort:heavy","configSlug":"gpt-5-4-heavy","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (heavy)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"heavy","reasoningCondition":"effort:heavy","isDefault":false,"n":1,"score":158.87,"scoreLo":154.11,"scoreHi":164.15,"reasoningOffset":2.59,"rankEligible":false},{"stateId":"gpt-5-4::effort:high","configSlug":"gpt-5-4-high","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (high)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":56,"score":157.64,"scoreLo":154.65,"scoreHi":161.16,"reasoningOffset":1.35,"rankEligible":true},{"stateId":"gpt-5-4::effort:low","configSlug":"gpt-5-4-low","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (low)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":26,"score":154.25,"scoreLo":151.58,"scoreHi":157.48,"reasoningOffset":-2.04,"rankEligible":true},{"stateId":"gpt-5-4::effort:medium","configSlug":"gpt-5-4-medium","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (medium)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":28,"score":156.84,"scoreLo":153.87,"scoreHi":160.46,"reasoningOffset":0.55,"rankEligible":true},{"stateId":"gpt-5-4-mini::default","configSlug":"gpt-5-4-mini","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini","modelName":"GPT-5.4 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":20,"score":150.71,"scoreLo":148.26,"scoreHi":153.55,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-4-mini::effort:high","configSlug":"gpt-5-4-mini-high","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini (high)","modelName":"GPT-5.4 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":34,"score":150.53,"scoreLo":148.17,"scoreHi":153.29,"reasoningOffset":-0.18,"rankEligible":true},{"stateId":"gpt-5-4-mini::effort:low","configSlug":"gpt-5-4-mini-low","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini (low)","modelName":"GPT-5.4 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":14,"score":145.41,"scoreLo":143.17,"scoreHi":147.84,"reasoningOffset":-5.3,"rankEligible":true},{"stateId":"gpt-5-4-mini::effort:medium","configSlug":"gpt-5-4-mini-medium","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini (medium)","modelName":"GPT-5.4 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":19,"score":148.88,"scoreLo":146.54,"scoreHi":151.61,"reasoningOffset":-1.83,"rankEligible":true},{"stateId":"gpt-5-4-mini::effort:minimal","configSlug":"gpt-5-4-mini-minimal","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini (minimal)","modelName":"GPT-5.4 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":145.92,"scoreLo":141.14,"scoreHi":150.69,"reasoningOffset":-4.79,"rankEligible":false},{"stateId":"gpt-5-4-mini::disabled","configSlug":"gpt-5-4-mini-no-thinking","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini (disabled)","modelName":"GPT-5.4 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":17,"score":141.04,"scoreLo":138.48,"scoreHi":143.63,"reasoningOffset":-9.67,"rankEligible":true},{"stateId":"gpt-5-4-mini::effort:xhigh","configSlug":"gpt-5-4-mini-xhigh","modelId":"gpt-5-4-mini","modelSlug":"gpt-5-4-mini","name":"GPT-5.4 Mini (xhigh)","modelName":"GPT-5.4 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":74,"score":151.6,"scoreLo":149.15,"scoreHi":154.41,"reasoningOffset":0.89,"rankEligible":true,"rankMean":114.4147,"rankLo":106.0,"rankHi":124.0},{"stateId":"gpt-5-4::effort:minimal","configSlug":"gpt-5-4-minimal","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (minimal)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":152.11,"scoreLo":147.51,"scoreHi":157.63,"reasoningOffset":-4.19,"rankEligible":false},{"stateId":"gpt-5-4-nano::default","configSlug":"gpt-5-4-nano","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano","modelName":"GPT-5.4 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":147.04,"scoreLo":144.8,"scoreHi":149.63,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-4-nano::effort:high","configSlug":"gpt-5-4-nano-high","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano (high)","modelName":"GPT-5.4 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":51,"score":146.72,"scoreLo":144.68,"scoreHi":149.07,"reasoningOffset":-0.32,"rankEligible":true},{"stateId":"gpt-5-4-nano::effort:low","configSlug":"gpt-5-4-nano-low","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano (low)","modelName":"GPT-5.4 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":13,"score":144.01,"scoreLo":141.82,"scoreHi":146.39,"reasoningOffset":-3.03,"rankEligible":true},{"stateId":"gpt-5-4-nano::effort:medium","configSlug":"gpt-5-4-nano-medium","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano (medium)","modelName":"GPT-5.4 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":15,"score":145.66,"scoreLo":143.5,"scoreHi":148.14,"reasoningOffset":-1.38,"rankEligible":true},{"stateId":"gpt-5-4-nano::effort:minimal","configSlug":"gpt-5-4-nano-minimal","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano (minimal)","modelName":"GPT-5.4 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":142.37,"scoreLo":137.9,"scoreHi":147.08,"reasoningOffset":-4.66,"rankEligible":false},{"stateId":"gpt-5-4-nano::disabled","configSlug":"gpt-5-4-nano-no-thinking","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano (disabled)","modelName":"GPT-5.4 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":15,"score":136.6,"scoreLo":133.97,"scoreHi":139.02,"reasoningOffset":-10.44,"rankEligible":true},{"stateId":"gpt-5-4-nano::effort:xhigh","configSlug":"gpt-5-4-nano-xhigh","modelId":"gpt-5-4-nano","modelSlug":"gpt-5-4-nano","name":"GPT-5.4 Nano (xhigh)","modelName":"GPT-5.4 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":40,"score":148.24,"scoreLo":145.99,"scoreHi":150.78,"reasoningOffset":1.2,"rankEligible":true,"rankMean":150.9689,"rankLo":139.0,"rankHi":164.0},{"stateId":"gpt-5-4::disabled","configSlug":"gpt-5-4-no-thinking","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (disabled)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":28,"score":147.7,"scoreLo":145.26,"scoreHi":150.27,"reasoningOffset":-8.6,"rankEligible":true},{"stateId":"gpt-5-4-pro::default","configSlug":"gpt-5-4-pro","modelId":"gpt-5-4-pro","modelSlug":"gpt-5-4-pro","name":"GPT-5.4 Pro","modelName":"GPT-5.4 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":160.06,"scoreLo":156.45,"scoreHi":164.29,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-4-pro::effort:extended","configSlug":"gpt-5-4-pro-extended-chatgpt-pro","modelId":"gpt-5-4-pro","modelSlug":"gpt-5-4-pro","name":"GPT-5.4 Pro (extended)","modelName":"GPT-5.4 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":2,"score":161.92,"scoreLo":158.06,"scoreHi":166.6,"reasoningOffset":1.86,"rankEligible":false},{"stateId":"gpt-5-4-pro::enabled","configSlug":"gpt-5-4-pro-taxbench","modelId":"gpt-5-4-pro","modelSlug":"gpt-5-4-pro","name":"GPT-5.4 Pro (enabled)","modelName":"GPT-5.4 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":161.27,"scoreLo":156.51,"scoreHi":166.94,"reasoningOffset":1.21,"rankEligible":false},{"stateId":"gpt-5-4-pro::effort:xhigh","configSlug":"gpt-5-4-pro-xhigh","modelId":"gpt-5-4-pro","modelSlug":"gpt-5-4-pro","name":"GPT-5.4 Pro (xhigh)","modelName":"GPT-5.4 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":17,"score":161.77,"scoreLo":158.25,"scoreHi":166.11,"reasoningOffset":1.72,"rankEligible":true,"rankMean":30.3794,"rankLo":21.0,"rankHi":43.0},{"stateId":"gpt-5-4::enabled","configSlug":"gpt-5-4-thinking","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (thinking)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":4,"score":158.49,"scoreLo":154.69,"scoreHi":162.95,"reasoningOffset":2.19,"rankEligible":true},{"stateId":"gpt-5-4::effort:xhigh","configSlug":"gpt-5-4-xhigh","modelId":"gpt-5-4","modelSlug":"gpt-5-4","name":"GPT-5.4 (xhigh)","modelName":"GPT-5.4","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":149,"score":158.77,"scoreLo":155.73,"scoreHi":162.39,"reasoningOffset":2.48,"rankEligible":true,"rankMean":62.2764,"rankLo":56.0,"rankHi":68.0},{"stateId":"gpt-5-5::default","configSlug":"gpt-5-5","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":101,"score":158.56,"scoreLo":155.55,"scoreHi":162.2,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-5::effort:heavy","configSlug":"gpt-5-5-heavy","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (heavy)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"heavy","reasoningCondition":"effort:heavy","isDefault":false,"n":1,"score":161.11,"scoreLo":156.11,"scoreHi":166.56,"reasoningOffset":2.55,"rankEligible":false},{"stateId":"gpt-5-5::effort:high","configSlug":"gpt-5-5-high","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (high)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":79,"score":160.23,"scoreLo":157.02,"scoreHi":164.03,"reasoningOffset":1.67,"rankEligible":true},{"stateId":"chat-latest-2026-05-05::default","configSlug":"gpt-5-5-instant-05-26","modelId":"chat-latest-2026-05-05","modelSlug":"chat-latest-2026-05-05","name":"GPT-5.5 Instant","modelName":"GPT-5.5 Instant","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":148.2,"scoreLo":145.14,"scoreHi":151.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":151.535,"rankLo":127.0,"rankHi":177.0},{"stateId":"gpt-5-5::effort:low","configSlug":"gpt-5-5-low","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (low)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":40,"score":155.36,"scoreLo":152.56,"scoreHi":158.67,"reasoningOffset":-3.2,"rankEligible":true},{"stateId":"gpt-5-5::effort:max","configSlug":"gpt-5-5-max","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (max)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":2,"score":159.85,"scoreLo":156.15,"scoreHi":164.41,"reasoningOffset":1.28,"rankEligible":false},{"stateId":"gpt-5-5::effort:medium","configSlug":"gpt-5-5-medium","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (medium)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":57,"score":158.93,"scoreLo":155.82,"scoreHi":162.64,"reasoningOffset":0.37,"rankEligible":true},{"stateId":"gpt-5-5::disabled","configSlug":"gpt-5-5-no-thinking","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (disabled)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":30,"score":149.58,"scoreLo":147.07,"scoreHi":152.42,"reasoningOffset":-8.98,"rankEligible":true},{"stateId":"gpt-5-5-pro::effort:extended","configSlug":"gpt-5-5-pro-extended-chatgpt-pro","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro (extended)","modelName":"GPT-5.5 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":2,"score":163.29,"scoreLo":159.06,"scoreHi":168.25,"reasoningOffset":1.64,"rankEligible":false},{"stateId":"gpt-5-5-pro::effort:high","configSlug":"gpt-5-5-pro-high","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro (high)","modelName":"GPT-5.5 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":162.79,"scoreLo":158.88,"scoreHi":167.37,"reasoningOffset":1.15,"rankEligible":false},{"stateId":"gpt-5-5-pro::effort:max","configSlug":"gpt-5-5-pro-max","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro (max)","modelName":"GPT-5.5 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":1,"score":162.81,"scoreLo":158.61,"scoreHi":167.9,"reasoningOffset":1.17,"rankEligible":false},{"stateId":"gpt-5-5-pro::effort:medium","configSlug":"gpt-5-5-pro-medium","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro (medium)","modelName":"GPT-5.5 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":161.44,"scoreLo":156.35,"scoreHi":167.15,"reasoningOffset":-0.2,"rankEligible":false},{"stateId":"gpt-5-5-pro::enabled","configSlug":"gpt-5-5-pro-taxbench","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro (enabled)","modelName":"GPT-5.5 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":162.89,"scoreLo":157.92,"scoreHi":168.38,"reasoningOffset":1.24,"rankEligible":false},{"stateId":"gpt-5-5-pro::default","configSlug":"gpt-5-5-pro-unknown","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro","modelName":"GPT-5.5 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":4,"score":161.64,"scoreLo":158.02,"scoreHi":166.02,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-5-pro::effort:xhigh","configSlug":"gpt-5-5-pro-xhigh","modelId":"gpt-5-5-pro","modelSlug":"gpt-5-5-pro","name":"GPT-5.5 Pro (xhigh)","modelName":"GPT-5.5 Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":18,"score":163.33,"scoreLo":159.58,"scoreHi":167.83,"reasoningOffset":1.69,"rankEligible":true,"rankMean":20.2056,"rankLo":16.0,"rankHi":27.0},{"stateId":"gpt-5-5::enabled","configSlug":"gpt-5-5-reasoning","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (enabled)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":5,"score":159.78,"scoreLo":156.1,"scoreHi":164.23,"reasoningOffset":1.22,"rankEligible":true},{"stateId":"gpt-5-5::effort:xhigh","configSlug":"gpt-5-5-xhigh","modelId":"gpt-5-5","modelSlug":"gpt-5-5","name":"GPT-5.5 (xhigh)","modelName":"GPT-5.5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":182,"score":161.16,"scoreLo":157.82,"scoreHi":165.13,"reasoningOffset":2.6,"rankEligible":true,"rankMean":36.3083,"rankLo":30.0,"rankHi":42.0},{"stateId":"gpt-5-6-luna::default","configSlug":"gpt-5-6-luna","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":64,"score":157.58,"scoreLo":154.59,"scoreHi":161.1,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-6-luna::effort:high","configSlug":"gpt-5-6-luna-high","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna (high)","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":57,"score":157.04,"scoreLo":154.13,"scoreHi":160.48,"reasoningOffset":-0.54,"rankEligible":true},{"stateId":"gpt-5-6-luna::effort:low","configSlug":"gpt-5-6-luna-low","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna (low)","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":44,"score":150.55,"scoreLo":148.19,"scoreHi":153.3,"reasoningOffset":-7.03,"rankEligible":true},{"stateId":"gpt-5-6-luna::effort:max","configSlug":"gpt-5-6-luna-max","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna (max)","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":120,"score":159.46,"scoreLo":156.3,"scoreHi":163.16,"reasoningOffset":1.88,"rankEligible":true,"rankMean":54.615,"rankLo":48.0,"rankHi":61.0},{"stateId":"gpt-5-6-luna::effort:medium","configSlug":"gpt-5-6-luna-medium","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna (medium)","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":93,"score":152.88,"scoreLo":150.33,"scoreHi":155.87,"reasoningOffset":-4.7,"rankEligible":true},{"stateId":"gpt-5-6-luna::effort:minimal","configSlug":"gpt-5-6-luna-minimal","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna (minimal)","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":153.37,"scoreLo":148.66,"scoreHi":158.75,"reasoningOffset":-4.21,"rankEligible":false},{"stateId":"gpt-5-6-luna::disabled","configSlug":"gpt-5-6-luna-no-thinking","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna (disabled)","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":32,"score":145.8,"scoreLo":143.41,"scoreHi":148.4,"reasoningOffset":-11.78,"rankEligible":true},{"stateId":"gpt-5-6-luna-pro::default","configSlug":"gpt-5-6-luna-pro","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":5,"score":160.71,"scoreLo":156.71,"scoreHi":165.33,"reasoningOffset":0.0,"rankEligible":true,"rankMean":41.5917,"rankLo":22.0,"rankHi":66.0},{"stateId":"gpt-5-6-luna-pro::enabled","configSlug":"gpt-5-6-luna-pro-ape-cred-v0-92","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro (enabled)","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":1,"score":161.7,"scoreLo":156.74,"scoreHi":167.52,"reasoningOffset":0.98,"rankEligible":false},{"stateId":"gpt-5-6-luna-pro::effort:high","configSlug":"gpt-5-6-luna-pro-high","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro (high)","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":161.19,"scoreLo":156.52,"scoreHi":166.47,"reasoningOffset":0.47,"rankEligible":false},{"stateId":"gpt-5-6-luna-pro::effort:low","configSlug":"gpt-5-6-luna-pro-low","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro (low)","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":158.69,"scoreLo":153.52,"scoreHi":164.45,"reasoningOffset":-2.02,"rankEligible":false},{"stateId":"gpt-5-6-luna-pro::effort:max","configSlug":"gpt-5-6-luna-pro-max","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro (max)","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":1,"score":161.38,"scoreLo":156.97,"scoreHi":166.49,"reasoningOffset":0.66,"rankEligible":false},{"stateId":"gpt-5-6-luna-pro::effort:medium","configSlug":"gpt-5-6-luna-pro-medium","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro (medium)","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":159.88,"scoreLo":154.7,"scoreHi":165.85,"reasoningOffset":-0.84,"rankEligible":false},{"stateId":"gpt-5-6-luna-pro::effort:minimal","configSlug":"gpt-5-6-luna-pro-minimal","modelId":"gpt-5-6-luna-pro","modelSlug":"gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro (minimal)","modelName":"GPT-5.6 Luna Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":156.1,"scoreLo":150.62,"scoreHi":162.18,"reasoningOffset":-4.61,"rankEligible":false},{"stateId":"gpt-5-6-luna::effort:xhigh","configSlug":"gpt-5-6-luna-xhigh","modelId":"gpt-5-6-luna","modelSlug":"gpt-5-6-luna","name":"GPT-5.6 Luna (xhigh)","modelName":"GPT-5.6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":52,"score":158.59,"scoreLo":155.45,"scoreHi":162.35,"reasoningOffset":1.02,"rankEligible":true},{"stateId":"gpt-5-6-sol::default","configSlug":"gpt-5-6-sol","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":110,"score":161.62,"scoreLo":158.27,"scoreHi":165.68,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-6-sol::effort:best-of-efforts","configSlug":"gpt-5-6-sol-best-of-efforts","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (best-of-efforts)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"best-of-efforts","reasoningCondition":"effort:best-of-efforts","isDefault":false,"n":10,"score":164.18,"scoreLo":160.29,"scoreHi":168.87,"reasoningOffset":2.56,"rankEligible":false},{"stateId":"gpt-5-6-sol::effort:high","configSlug":"gpt-5-6-sol-high","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (high)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":78,"score":162.42,"scoreLo":158.93,"scoreHi":166.55,"reasoningOffset":0.8,"rankEligible":true},{"stateId":"gpt-5-6-sol::effort:low","configSlug":"gpt-5-6-sol-low","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (low)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":53,"score":157.94,"scoreLo":154.9,"scoreHi":161.51,"reasoningOffset":-3.68,"rankEligible":true},{"stateId":"gpt-5-6-sol::effort:max","configSlug":"gpt-5-6-sol-max","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (max)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":156,"score":163.97,"scoreLo":160.37,"scoreHi":168.37,"reasoningOffset":2.35,"rankEligible":true,"rankMean":18.0175,"rankLo":16.0,"rankHi":20.0},{"stateId":"gpt-5-6-sol::effort:medium","configSlug":"gpt-5-6-sol-medium","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (medium)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":64,"score":161.0,"scoreLo":157.64,"scoreHi":165.05,"reasoningOffset":-0.62,"rankEligible":true},{"stateId":"gpt-5-6-sol::effort:minimal","configSlug":"gpt-5-6-sol-minimal","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (minimal)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":157.42,"scoreLo":152.36,"scoreHi":163.43,"reasoningOffset":-4.2,"rankEligible":false},{"stateId":"gpt-5-6-sol::disabled","configSlug":"gpt-5-6-sol-no-thinking","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (disabled)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":27,"score":153.87,"scoreLo":150.98,"scoreHi":157.28,"reasoningOffset":-7.75,"rankEligible":true},{"stateId":"gpt-5-6-sol-pro::default","configSlug":"gpt-5-6-sol-pro","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":9,"score":162.49,"scoreLo":158.71,"scoreHi":166.97,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-6-sol-pro::effort:extended","configSlug":"gpt-5-6-sol-pro-extended-genebench-tools","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro (extended)","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"extended","reasoningCondition":"effort:extended","isDefault":false,"n":1,"score":164.65,"scoreLo":160.39,"scoreHi":169.66,"reasoningOffset":2.16,"rankEligible":false},{"stateId":"gpt-5-6-sol-pro::effort:high","configSlug":"gpt-5-6-sol-pro-high","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro (high)","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":162.99,"scoreLo":158.41,"scoreHi":168.2,"reasoningOffset":0.49,"rankEligible":false},{"stateId":"gpt-5-6-sol-pro::effort:low","configSlug":"gpt-5-6-sol-pro-low","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro (low)","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":160.48,"scoreLo":155.24,"scoreHi":166.35,"reasoningOffset":-2.0,"rankEligible":false},{"stateId":"gpt-5-6-sol-pro::effort:max","configSlug":"gpt-5-6-sol-pro-max","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro (max)","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":30,"score":164.63,"scoreLo":160.76,"scoreHi":169.36,"reasoningOffset":2.14,"rankEligible":true,"rankMean":15.4581,"rankLo":10.0,"rankHi":20.0},{"stateId":"gpt-5-6-sol-pro::effort:medium","configSlug":"gpt-5-6-sol-pro-medium","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro (medium)","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":162.23,"scoreLo":157.07,"scoreHi":168.09,"reasoningOffset":-0.26,"rankEligible":false},{"stateId":"gpt-5-6-sol-pro::effort:minimal","configSlug":"gpt-5-6-sol-pro-minimal","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro (minimal)","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":158.27,"scoreLo":152.99,"scoreHi":164.22,"reasoningOffset":-4.22,"rankEligible":false},{"stateId":"gpt-5-6-sol-pro::effort:xhigh","configSlug":"gpt-5-6-sol-pro-xhigh-simplebench","modelId":"gpt-5-6-sol-pro","modelSlug":"gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro (xhigh)","modelName":"GPT-5.6 Sol Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":163.14,"scoreLo":158.75,"scoreHi":168.2,"reasoningOffset":0.65,"rankEligible":false},{"stateId":"gpt-5-6-sol::enabled","configSlug":"gpt-5-6-sol-proximus","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (enabled)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":2,"score":162.34,"scoreLo":158.15,"scoreHi":167.26,"reasoningOffset":0.72,"rankEligible":false},{"stateId":"gpt-5-6-sol::effort:xhigh","configSlug":"gpt-5-6-sol-xhigh","modelId":"gpt-5-6-sol","modelSlug":"gpt-5-6-sol","name":"GPT-5.6 Sol (xhigh)","modelName":"GPT-5.6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":81,"score":163.22,"scoreLo":159.68,"scoreHi":167.49,"reasoningOffset":1.6,"rankEligible":true},{"stateId":"gpt-5-6-terra::default","configSlug":"gpt-5-6-terra","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":43,"score":159.26,"scoreLo":156.11,"scoreHi":162.94,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-6-terra::effort:high","configSlug":"gpt-5-6-terra-high","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra (high)","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":59,"score":159.0,"scoreLo":155.91,"scoreHi":162.71,"reasoningOffset":-0.26,"rankEligible":true},{"stateId":"gpt-5-6-terra::effort:low","configSlug":"gpt-5-6-terra-low","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra (low)","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":45,"score":154.97,"scoreLo":152.2,"scoreHi":158.29,"reasoningOffset":-4.29,"rankEligible":true},{"stateId":"gpt-5-6-terra::effort:max","configSlug":"gpt-5-6-terra-max","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra (max)","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":118,"score":161.87,"scoreLo":158.43,"scoreHi":165.96,"reasoningOffset":2.61,"rankEligible":true,"rankMean":29.0,"rankLo":24.0,"rankHi":35.0},{"stateId":"gpt-5-6-terra::effort:medium","configSlug":"gpt-5-6-terra-medium","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra (medium)","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":49,"score":156.82,"scoreLo":153.9,"scoreHi":160.4,"reasoningOffset":-2.44,"rankEligible":true},{"stateId":"gpt-5-6-terra::effort:minimal","configSlug":"gpt-5-6-terra-minimal","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra (minimal)","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":155.06,"scoreLo":150.06,"scoreHi":160.62,"reasoningOffset":-4.2,"rankEligible":false},{"stateId":"gpt-5-6-terra::disabled","configSlug":"gpt-5-6-terra-no-thinking","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra (disabled)","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":29,"score":149.85,"scoreLo":147.31,"scoreHi":152.77,"reasoningOffset":-9.42,"rankEligible":true},{"stateId":"gpt-5-6-terra-pro::default","configSlug":"gpt-5-6-terra-pro","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro","modelName":"GPT-5.6 Terra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":5,"score":162.36,"scoreLo":158.03,"scoreHi":167.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":27.2925,"rankLo":15.0,"rankHi":51.0},{"stateId":"gpt-5-6-terra-pro::effort:high","configSlug":"gpt-5-6-terra-pro-high","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro (high)","modelName":"GPT-5.6 Terra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":162.93,"scoreLo":158.07,"scoreHi":168.5,"reasoningOffset":0.57,"rankEligible":false},{"stateId":"gpt-5-6-terra-pro::effort:low","configSlug":"gpt-5-6-terra-pro-low","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro (low)","modelName":"GPT-5.6 Terra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":160.52,"scoreLo":154.95,"scoreHi":166.78,"reasoningOffset":-1.85,"rankEligible":false},{"stateId":"gpt-5-6-terra-pro::effort:medium","configSlug":"gpt-5-6-terra-pro-medium","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro (medium)","modelName":"GPT-5.6 Terra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":161.91,"scoreLo":156.51,"scoreHi":168.17,"reasoningOffset":-0.45,"rankEligible":false},{"stateId":"gpt-5-6-terra-pro::effort:minimal","configSlug":"gpt-5-6-terra-pro-minimal","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro (minimal)","modelName":"GPT-5.6 Terra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":158.14,"scoreLo":152.5,"scoreHi":164.58,"reasoningOffset":-4.22,"rankEligible":false},{"stateId":"gpt-5-6-terra-pro::effort:xhigh","configSlug":"gpt-5-6-terra-pro-xhigh","modelId":"gpt-5-6-terra-pro","modelSlug":"gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro (xhigh)","modelName":"GPT-5.6 Terra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":163.15,"scoreLo":158.3,"scoreHi":168.67,"reasoningOffset":0.79,"rankEligible":false},{"stateId":"gpt-5-6-terra::effort:xhigh","configSlug":"gpt-5-6-terra-xhigh","modelId":"gpt-5-6-terra","modelSlug":"gpt-5-6-terra","name":"GPT-5.6 Terra (xhigh)","modelName":"GPT-5.6 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":65,"score":160.34,"scoreLo":157.08,"scoreHi":164.25,"reasoningOffset":1.08,"rankEligible":true},{"stateId":"gpt-5-chat-latest::default","configSlug":"gpt-5-chat-latest","modelId":"gpt-5-chat-latest","modelSlug":"gpt-5-chat-latest","name":"GPT-5 Chat","modelName":"GPT-5 Chat","organizationSlug":"openai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":137.9,"scoreLo":135.39,"scoreHi":140.53,"reasoningOffset":0.0,"rankEligible":true,"rankMean":268.5417,"rankLo":238.0,"rankHi":298.0},{"stateId":"gpt-5-codex::default","configSlug":"gpt-5-codex","modelId":"gpt-5-codex","modelSlug":"gpt-5-codex","name":"GPT-5-Codex","modelName":"GPT-5-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":148.47,"scoreLo":145.35,"scoreHi":152.04,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-codex::effort:high","configSlug":"gpt-5-codex-high","modelId":"gpt-5-codex","modelSlug":"gpt-5-codex","name":"GPT-5-Codex (high)","modelName":"GPT-5-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":13,"score":149.94,"scoreLo":146.95,"scoreHi":153.28,"reasoningOffset":1.46,"rankEligible":true,"rankMean":132.4039,"rankLo":110.0,"rankHi":156.0},{"stateId":"gpt-5-codex::enabled","configSlug":"gpt-5-codex-snakebench","modelId":"gpt-5-codex","modelSlug":"gpt-5-codex","name":"GPT-5-Codex (enabled)","modelName":"GPT-5-Codex","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":149.7,"scoreLo":145.31,"scoreHi":154.5,"reasoningOffset":1.23,"rankEligible":false},{"stateId":"gpt-5::effort:high","configSlug":"gpt-5-high","modelId":"gpt-5","modelSlug":"gpt-5","name":"GPT-5 (high)","modelName":"GPT-5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":129,"score":150.06,"scoreLo":147.94,"scoreHi":152.54,"reasoningOffset":0.73,"rankEligible":true},{"stateId":"gpt-5::effort:low","configSlug":"gpt-5-low","modelId":"gpt-5","modelSlug":"gpt-5","name":"GPT-5 (low)","modelName":"GPT-5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":20,"score":148.0,"scoreLo":145.84,"scoreHi":150.38,"reasoningOffset":-1.32,"rankEligible":true},{"stateId":"gpt-5::effort:medium","configSlug":"gpt-5-medium","modelId":"gpt-5","modelSlug":"gpt-5","name":"GPT-5 (medium)","modelName":"GPT-5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":33,"score":149.77,"scoreLo":147.6,"scoreHi":152.34,"reasoningOffset":0.45,"rankEligible":true},{"stateId":"gpt-5-mini::default","configSlug":"gpt-5-mini","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini","modelName":"GPT-5 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":105,"score":144.43,"scoreLo":142.57,"scoreHi":146.47,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-mini::effort:high","configSlug":"gpt-5-mini-high","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini (high)","modelName":"GPT-5 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":112,"score":145.71,"scoreLo":143.84,"scoreHi":147.83,"reasoningOffset":1.28,"rankEligible":true,"rankMean":178.5128,"rankLo":170.0,"rankHi":188.0},{"stateId":"gpt-5-mini::effort:low","configSlug":"gpt-5-mini-low","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini (low)","modelName":"GPT-5 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":10,"score":143.0,"scoreLo":140.87,"scoreHi":145.27,"reasoningOffset":-1.43,"rankEligible":true},{"stateId":"gpt-5-mini::effort:medium","configSlug":"gpt-5-mini-medium","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini (medium)","modelName":"GPT-5 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":20,"score":144.7,"scoreLo":142.71,"scoreHi":146.94,"reasoningOffset":0.27,"rankEligible":true},{"stateId":"gpt-5-mini::effort:minimal","configSlug":"gpt-5-mini-minimal","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini (minimal)","modelName":"GPT-5 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":45,"score":138.13,"scoreLo":136.46,"scoreHi":139.83,"reasoningOffset":-6.3,"rankEligible":true},{"stateId":"gpt-5-mini::enabled","configSlug":"gpt-5-mini-thinking","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini (thinking)","modelName":"GPT-5 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":145.11,"scoreLo":141.61,"scoreHi":148.73,"reasoningOffset":0.67,"rankEligible":true},{"stateId":"gpt-5-mini::effort:xhigh","configSlug":"gpt-5-mini-xhigh","modelId":"gpt-5-mini","modelSlug":"gpt-5-mini","name":"GPT-5 Mini (xhigh)","modelName":"GPT-5 Mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":145.15,"scoreLo":142.27,"scoreHi":148.32,"reasoningOffset":0.72,"rankEligible":false},{"stateId":"gpt-5::effort:minimal","configSlug":"gpt-5-minimal","modelId":"gpt-5","modelSlug":"gpt-5","name":"GPT-5 (minimal)","modelName":"GPT-5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":51,"score":142.14,"scoreLo":140.44,"scoreHi":144.04,"reasoningOffset":-7.18,"rankEligible":true},{"stateId":"gpt-5-nano::default","configSlug":"gpt-5-nano","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano","modelName":"GPT-5 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":60,"score":133.84,"scoreLo":131.94,"scoreHi":135.67,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-5-nano::enabled","configSlug":"gpt-5-nano-2025-08-07-5x-ensemble","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano (enabled)","modelName":"GPT-5 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":5,"score":134.29,"scoreLo":130.74,"scoreHi":137.78,"reasoningOffset":0.45,"rankEligible":true},{"stateId":"gpt-5-nano::effort:high","configSlug":"gpt-5-nano-high","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano (high)","modelName":"GPT-5 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":88,"score":135.78,"scoreLo":134.17,"scoreHi":137.36,"reasoningOffset":1.94,"rankEligible":true},{"stateId":"gpt-5-nano::effort:low","configSlug":"gpt-5-nano-low","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano (low)","modelName":"GPT-5 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":13,"score":133.11,"scoreLo":130.96,"scoreHi":135.14,"reasoningOffset":-0.74,"rankEligible":true},{"stateId":"gpt-5-nano::effort:medium","configSlug":"gpt-5-nano-medium","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano (medium)","modelName":"GPT-5 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":19,"score":136.51,"scoreLo":134.55,"scoreHi":138.36,"reasoningOffset":2.67,"rankEligible":true,"rankMean":287.4592,"rankLo":268.0,"rankHi":307.0},{"stateId":"gpt-5-nano::effort:minimal","configSlug":"gpt-5-nano-minimal","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano (minimal)","modelName":"GPT-5 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":41,"score":125.06,"scoreLo":122.27,"scoreHi":127.41,"reasoningOffset":-8.79,"rankEligible":true},{"stateId":"gpt-5-nano::effort:xhigh","configSlug":"gpt-5-nano-xhigh","modelId":"gpt-5-nano","modelSlug":"gpt-5-nano","name":"GPT-5 Nano (xhigh)","modelName":"GPT-5 Nano","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":134.31,"scoreLo":131.38,"scoreHi":137.14,"reasoningOffset":0.46,"rankEligible":false},{"stateId":"gpt-5-pro::effort:high","configSlug":"gpt-5-pro-high","modelId":"gpt-5-pro","modelSlug":"gpt-5-pro","name":"GPT-5 Pro (high)","modelName":"GPT-5 Pro","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":18,"score":152.98,"scoreLo":150.07,"scoreHi":156.28,"reasoningOffset":0.51,"rankEligible":true,"rankMean":102.1481,"rankLo":87.0,"rankHi":118.0},{"stateId":"gpt-5::enabled","configSlug":"gpt-5-thinking","modelId":"gpt-5","modelSlug":"gpt-5","name":"GPT-5 (thinking)","modelName":"GPT-5","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":150.0,"scoreLo":150.0,"scoreHi":150.0,"reasoningOffset":0.74,"rankEligible":true,"rankMean":130.9883,"rankLo":108.0,"rankHi":155.025},{"stateId":"gpt-6-1-sol::default","configSlug":"gpt-6-1-sol","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol","modelName":"GPT-6.1 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":43,"score":164.47,"scoreLo":160.75,"scoreHi":169.02,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-6-1-sol::enabled","configSlug":"gpt-6-1-sol-concept-synth-cascade","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol (enabled)","modelName":"GPT-6.1 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":168.39,"scoreLo":163.59,"scoreHi":174.16,"reasoningOffset":3.92,"rankEligible":false},{"stateId":"gpt-6-1-sol::effort:high","configSlug":"gpt-6-1-sol-high","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol (high)","modelName":"GPT-6.1 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":43,"score":166.19,"scoreLo":162.3,"scoreHi":170.88,"reasoningOffset":1.71,"rankEligible":true},{"stateId":"gpt-6-1-sol::effort:low","configSlug":"gpt-6-1-sol-low","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol (low)","modelName":"GPT-6.1 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":28,"score":162.82,"scoreLo":159.25,"scoreHi":167.12,"reasoningOffset":-1.66,"rankEligible":true},{"stateId":"gpt-6-1-sol::effort:max","configSlug":"gpt-6-1-sol-max","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol (max)","modelName":"GPT-6.1 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":119,"score":167.31,"scoreLo":163.26,"scoreHi":172.09,"reasoningOffset":2.83,"rankEligible":true,"rankMean":5.9264,"rankLo":5.0,"rankHi":8.0},{"stateId":"gpt-6-1-sol::effort:medium","configSlug":"gpt-6-1-sol-medium","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol (medium)","modelName":"GPT-6.1 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":32,"score":164.96,"scoreLo":161.09,"scoreHi":169.5,"reasoningOffset":0.48,"rankEligible":true},{"stateId":"gpt-6-1-sol::effort:xhigh","configSlug":"gpt-6-1-sol-xhigh","modelId":"gpt-6-1-sol","modelSlug":"gpt-6-1-sol","name":"GPT-6.1 Sol (xhigh)","modelName":"GPT-6.1 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":34,"score":166.69,"scoreLo":162.68,"scoreHi":171.63,"reasoningOffset":2.21,"rankEligible":true},{"stateId":"gpt-6-1-terra::default","configSlug":"gpt-6-1-terra","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra","modelName":"GPT-6.1 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":163.47,"scoreLo":159.8,"scoreHi":167.8,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-6-1-terra::effort:high","configSlug":"gpt-6-1-terra-high","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra (high)","modelName":"GPT-6.1 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":16,"score":164.85,"scoreLo":160.94,"scoreHi":169.48,"reasoningOffset":1.38,"rankEligible":true},{"stateId":"gpt-6-1-terra::effort:low","configSlug":"gpt-6-1-terra-low","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra (low)","modelName":"GPT-6.1 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":14,"score":161.31,"scoreLo":157.9,"scoreHi":165.58,"reasoningOffset":-2.16,"rankEligible":true},{"stateId":"gpt-6-1-terra::effort:max","configSlug":"gpt-6-1-terra-max","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra (max)","modelName":"GPT-6.1 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":36,"score":166.21,"scoreLo":162.21,"scoreHi":170.99,"reasoningOffset":2.74,"rankEligible":true,"rankMean":9.4775,"rankLo":7.0,"rankHi":12.0},{"stateId":"gpt-6-1-terra::effort:medium","configSlug":"gpt-6-1-terra-medium","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra (medium)","modelName":"GPT-6.1 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":16,"score":163.6,"scoreLo":159.93,"scoreHi":168.05,"reasoningOffset":0.13,"rankEligible":true},{"stateId":"gpt-6-1-terra::disabled","configSlug":"gpt-6-1-terra-no-thinking","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra (disabled)","modelName":"GPT-6.1 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":11,"score":159.99,"scoreLo":156.39,"scoreHi":164.23,"reasoningOffset":-3.48,"rankEligible":true},{"stateId":"gpt-6-1-terra::effort:xhigh","configSlug":"gpt-6-1-terra-xhigh","modelId":"gpt-6-1-terra","modelSlug":"gpt-6-1-terra","name":"GPT-6.1 Terra (xhigh)","modelName":"GPT-6.1 Terra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":15,"score":165.22,"scoreLo":161.41,"scoreHi":169.81,"reasoningOffset":1.75,"rankEligible":true},{"stateId":"gpt-6-astra::default","configSlug":"gpt-6-astra","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":81,"score":165.49,"scoreLo":161.67,"scoreHi":170.09,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-6-astra::effort:best-of-efforts","configSlug":"gpt-6-astra-best-of-efforts","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra (best-of-efforts)","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"best-of-efforts","reasoningCondition":"effort:best-of-efforts","isDefault":false,"n":12,"score":168.15,"scoreLo":163.89,"scoreHi":173.36,"reasoningOffset":2.66,"rankEligible":false},{"stateId":"gpt-6-astra::enabled","configSlug":"gpt-6-astra-chatgpt-ultra","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra (enabled)","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"unknown","reasoningCondition":"enabled","isDefault":false,"n":2,"score":169.83,"scoreLo":165.03,"scoreHi":175.87,"reasoningOffset":4.34,"rankEligible":false},{"stateId":"gpt-6-astra::effort:high","configSlug":"gpt-6-astra-high","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra (high)","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":73,"score":167.37,"scoreLo":163.31,"scoreHi":172.15,"reasoningOffset":1.89,"rankEligible":true},{"stateId":"gpt-6-astra::effort:low","configSlug":"gpt-6-astra-low","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra (low)","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":39,"score":164.63,"scoreLo":160.93,"scoreHi":169.22,"reasoningOffset":-0.85,"rankEligible":true},{"stateId":"gpt-6-astra::effort:max","configSlug":"gpt-6-astra-max","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra (max)","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":140,"score":168.24,"scoreLo":164.1,"scoreHi":173.18,"reasoningOffset":2.75,"rankEligible":true,"rankMean":3.6394,"rankLo":2.0,"rankHi":5.0},{"stateId":"gpt-6-astra::effort:medium","configSlug":"gpt-6-astra-medium","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra (medium)","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":46,"score":166.32,"scoreLo":162.37,"scoreHi":171.03,"reasoningOffset":0.84,"rankEligible":true},{"stateId":"gpt-6-astra-pro::default","configSlug":"gpt-6-astra-pro","modelId":"gpt-6-astra-pro","modelSlug":"gpt-6-astra-pro","name":"GPT-6 Astra Pro","modelName":"GPT-6 Astra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":168.66,"scoreLo":163.14,"scoreHi":175.58,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"gpt-6-astra-pro::effort:max","configSlug":"gpt-6-astra-pro-max","modelId":"gpt-6-astra-pro","modelSlug":"gpt-6-astra-pro","name":"GPT-6 Astra Pro (max)","modelName":"GPT-6 Astra Pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":6,"score":170.07,"scoreLo":164.8,"scoreHi":176.7,"reasoningOffset":1.42,"rankEligible":true,"rankMean":1.7764,"rankLo":1.0,"rankHi":7.0},{"stateId":"gpt-6-astra::effort:xhigh","configSlug":"gpt-6-astra-xhigh","modelId":"gpt-6-astra","modelSlug":"gpt-6-astra","name":"GPT-6 Astra (xhigh)","modelName":"GPT-6 Astra","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":63,"score":167.59,"scoreLo":163.56,"scoreHi":172.53,"reasoningOffset":2.11,"rankEligible":true},{"stateId":"gpt-6-luna::default","configSlug":"gpt-6-luna","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna","modelName":"GPT-6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":32,"score":158.71,"scoreLo":155.62,"scoreHi":162.39,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-6-luna::effort:high","configSlug":"gpt-6-luna-high","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna (high)","modelName":"GPT-6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":37,"score":157.37,"scoreLo":154.41,"scoreHi":160.91,"reasoningOffset":-1.33,"rankEligible":true},{"stateId":"gpt-6-luna::effort:low","configSlug":"gpt-6-luna-low","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna (low)","modelName":"GPT-6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":26,"score":152.3,"scoreLo":149.7,"scoreHi":155.29,"reasoningOffset":-6.41,"rankEligible":true},{"stateId":"gpt-6-luna::effort:max","configSlug":"gpt-6-luna-max","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna (max)","modelName":"GPT-6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":111,"score":159.9,"scoreLo":156.7,"scoreHi":163.68,"reasoningOffset":1.19,"rankEligible":true,"rankMean":49.9075,"rankLo":44.0,"rankHi":56.0},{"stateId":"gpt-6-luna::effort:medium","configSlug":"gpt-6-luna-medium","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna (medium)","modelName":"GPT-6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":28,"score":156.06,"scoreLo":153.17,"scoreHi":159.47,"reasoningOffset":-2.65,"rankEligible":true},{"stateId":"gpt-6-luna::disabled","configSlug":"gpt-6-luna-no-thinking","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna (disabled)","modelName":"GPT-6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":20,"score":148.03,"scoreLo":145.4,"scoreHi":150.94,"reasoningOffset":-10.68,"rankEligible":true},{"stateId":"gpt-6-luna::effort:xhigh","configSlug":"gpt-6-luna-xhigh","modelId":"gpt-6-luna","modelSlug":"gpt-6-luna","name":"GPT-6 Luna (xhigh)","modelName":"GPT-6 Luna","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":32,"score":158.64,"scoreLo":155.53,"scoreHi":162.37,"reasoningOffset":-0.06,"rankEligible":true},{"stateId":"gpt-6-sol::default","configSlug":"gpt-6-sol","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol","modelName":"GPT-6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":37,"score":162.47,"scoreLo":158.98,"scoreHi":166.58,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-6-sol::effort:high","configSlug":"gpt-6-sol-high","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol (high)","modelName":"GPT-6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":40,"score":162.83,"scoreLo":159.32,"scoreHi":167.13,"reasoningOffset":0.36,"rankEligible":true},{"stateId":"gpt-6-sol::effort:low","configSlug":"gpt-6-sol-low","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol (low)","modelName":"GPT-6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":28,"score":158.72,"scoreLo":155.56,"scoreHi":162.48,"reasoningOffset":-3.75,"rankEligible":true},{"stateId":"gpt-6-sol::effort:max","configSlug":"gpt-6-sol-max","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol (max)","modelName":"GPT-6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":103,"score":164.9,"scoreLo":161.22,"scoreHi":169.4,"reasoningOffset":2.43,"rankEligible":true,"rankMean":14.5261,"rankLo":12.0,"rankHi":17.0},{"stateId":"gpt-6-sol::effort:medium","configSlug":"gpt-6-sol-medium","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol (medium)","modelName":"GPT-6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":32,"score":161.62,"scoreLo":158.23,"scoreHi":165.76,"reasoningOffset":-0.85,"rankEligible":true},{"stateId":"gpt-6-sol::disabled","configSlug":"gpt-6-sol-no-thinking","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol (disabled)","modelName":"GPT-6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":19,"score":154.57,"scoreLo":151.71,"scoreHi":157.85,"reasoningOffset":-7.9,"rankEligible":true},{"stateId":"gpt-6-sol::effort:xhigh","configSlug":"gpt-6-sol-xhigh","modelId":"gpt-6-sol","modelSlug":"gpt-6-sol","name":"GPT-6 Sol (xhigh)","modelName":"GPT-6 Sol","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":32,"score":163.79,"scoreLo":160.17,"scoreHi":168.14,"reasoningOffset":1.32,"rankEligible":true},{"stateId":"gpt-neox-20b::default","configSlug":"gpt-neox-20b","modelId":"gpt-neox-20b","modelSlug":"gpt-neox-20b","name":"GPT-NeoX-20B","modelName":"GPT-NeoX-20B","organizationSlug":"eleutherai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":67.82,"scoreLo":55.2,"scoreHi":77.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":863.1642,"rankLo":853.0,"rankHi":871.0},{"stateId":"gpt-oss-120b::default","configSlug":"gpt-oss-120b","modelId":"gpt-oss-120b","modelSlug":"gpt-oss-120b","name":"gpt-oss-120b","modelName":"gpt-oss-120b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":80,"score":137.79,"scoreLo":136.19,"scoreHi":139.47,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-oss-120b::effort:high","configSlug":"gpt-oss-120b-high","modelId":"gpt-oss-120b","modelSlug":"gpt-oss-120b","name":"gpt-oss-120b (high)","modelName":"gpt-oss-120b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":66,"score":140.76,"scoreLo":139.16,"scoreHi":142.48,"reasoningOffset":2.97,"rankEligible":true,"rankMean":229.5903,"rankLo":218.0,"rankHi":242.0},{"stateId":"gpt-oss-120b::effort:low","configSlug":"gpt-oss-120b-low","modelId":"gpt-oss-120b","modelSlug":"gpt-oss-120b","name":"gpt-oss-120b (low)","modelName":"gpt-oss-120b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":25,"score":136.37,"scoreLo":134.45,"scoreHi":138.26,"reasoningOffset":-1.42,"rankEligible":true},{"stateId":"gpt-oss-120b::effort:medium","configSlug":"gpt-oss-120b-medium","modelId":"gpt-oss-120b","modelSlug":"gpt-oss-120b","name":"gpt-oss-120b (medium)","modelName":"gpt-oss-120b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":18,"score":138.81,"scoreLo":136.83,"scoreHi":140.91,"reasoningOffset":1.02,"rankEligible":true},{"stateId":"gpt-oss-120b::effort:minimal","configSlug":"gpt-oss-120b-minimal","modelId":"gpt-oss-120b","modelSlug":"gpt-oss-120b","name":"gpt-oss-120b (minimal)","modelName":"gpt-oss-120b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":133.76,"scoreLo":129.18,"scoreHi":138.67,"reasoningOffset":-4.03,"rankEligible":false},{"stateId":"gpt-oss-120b::effort:xhigh","configSlug":"gpt-oss-120b-xhigh","modelId":"gpt-oss-120b","modelSlug":"gpt-oss-120b","name":"gpt-oss-120b (xhigh)","modelName":"gpt-oss-120b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":138.42,"scoreLo":135.72,"scoreHi":141.16,"reasoningOffset":0.63,"rankEligible":false},{"stateId":"gpt-oss-20b::default","configSlug":"gpt-oss-20b","modelId":"gpt-oss-20b","modelSlug":"gpt-oss-20b","name":"gpt-oss-20b","modelName":"gpt-oss-20b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":48,"score":131.6,"scoreLo":129.69,"scoreHi":133.42,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"gpt-oss-20b::effort:high","configSlug":"gpt-oss-20b-high","modelId":"gpt-oss-20b","modelSlug":"gpt-oss-20b","name":"gpt-oss-20b (high)","modelName":"gpt-oss-20b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":43,"score":134.84,"scoreLo":133.14,"scoreHi":136.58,"reasoningOffset":3.23,"rankEligible":true,"rankMean":310.1978,"rankLo":293.0,"rankHi":328.0},{"stateId":"gpt-oss-20b::effort:low","configSlug":"gpt-oss-20b-low","modelId":"gpt-oss-20b","modelSlug":"gpt-oss-20b","name":"gpt-oss-20b (low)","modelName":"gpt-oss-20b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":24,"score":130.64,"scoreLo":128.51,"scoreHi":132.67,"reasoningOffset":-0.96,"rankEligible":true},{"stateId":"gpt-oss-20b::effort:medium","configSlug":"gpt-oss-20b-medium","modelId":"gpt-oss-20b","modelSlug":"gpt-oss-20b","name":"gpt-oss-20b (medium)","modelName":"gpt-oss-20b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":17,"score":133.27,"scoreLo":131.19,"scoreHi":135.29,"reasoningOffset":1.67,"rankEligible":true},{"stateId":"gpt-oss-20b::effort:minimal","configSlug":"gpt-oss-20b-minimal","modelId":"gpt-oss-20b","modelSlug":"gpt-oss-20b","name":"gpt-oss-20b (minimal)","modelName":"gpt-oss-20b","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":127.51,"scoreLo":122.6,"scoreHi":132.36,"reasoningOffset":-4.09,"rankEligible":false},{"stateId":"granite-3-3-8b-instruct::default","configSlug":"granite-3-3-8b-instruct-helm-capabilities","modelId":"granite-3-3-8b-instruct","modelSlug":"granite-3-3-8b-instruct","name":"Granite 3.3 8B Instruct","modelName":"Granite 3.3 8B Instruct","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":5,"score":108.26,"scoreLo":101.23,"scoreHi":114.5,"reasoningOffset":0.0,"rankEligible":true,"rankMean":642.0356,"rankLo":582.0,"rankHi":695.0},{"stateId":"granite-3-3-8b-instruct::disabled","configSlug":"granite-3-3-8b-instruct-no-thinking","modelId":"granite-3-3-8b-instruct","modelSlug":"granite-3-3-8b-instruct","name":"Granite 3.3 8B Instruct (disabled)","modelName":"Granite 3.3 8B Instruct","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":107.29,"scoreLo":100.41,"scoreHi":113.25,"reasoningOffset":-0.97,"rankEligible":true},{"stateId":"granite-4-0-1b::default","configSlug":"granite-4-0-1b","modelId":"granite-4-0-1b","modelSlug":"granite-4-0-1b","name":"Granite 4.0 1B","modelName":"Granite 4.0 1B","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":99.4,"scoreLo":90.3,"scoreHi":106.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":723.0094,"rankLo":672.0,"rankHi":774.0},{"stateId":"granite-4-0-350m::default","configSlug":"granite-4-0-350m","modelId":"granite-4-0-350m","modelSlug":"granite-4-0-350m","name":"Granite 4.0 350M","modelName":"Granite 4.0 350M","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":84.51,"scoreLo":73.26,"scoreHi":94.18,"reasoningOffset":0.0,"rankEligible":true,"rankMean":815.1472,"rankLo":781.0,"rankHi":845.0},{"stateId":"granite-4-0-h-1b::default","configSlug":"granite-4-0-h-1b","modelId":"granite-4-0-h-1b","modelSlug":"granite-4-0-h-1b","name":"Granite 4.0 H 1B","modelName":"Granite 4.0 H 1B","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":99.45,"scoreLo":89.55,"scoreHi":107.88,"reasoningOffset":0.0,"rankEligible":true,"rankMean":722.2711,"rankLo":664.0,"rankHi":780.0},{"stateId":"granite-4-0-h-350m::default","configSlug":"granite-4-0-h-350m","modelId":"granite-4-0-h-350m","modelSlug":"granite-4-0-h-350m","name":"Granite 4.0 H 350M","modelName":"Granite 4.0 H 350M","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":85.29,"scoreLo":74.03,"scoreHi":94.53,"reasoningOffset":0.0,"rankEligible":true,"rankMean":811.8808,"rankLo":776.0,"rankHi":841.0},{"stateId":"granite-4-0-h-small::default","configSlug":"granite-4-0-h-small","modelId":"granite-4-0-h-small","modelSlug":"granite-4-0-h-small","name":"Granite 4.0 H Small","modelName":"Granite 4.0 H Small","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":120.18,"scoreLo":115.12,"scoreHi":124.49,"reasoningOffset":0.0,"rankEligible":true,"rankMean":499.6189,"rankLo":447.975,"rankHi":550.0},{"stateId":"granite-4-0-micro::default","configSlug":"granite-4-0-micro","modelId":"granite-4-0-micro","modelSlug":"granite-4-0-micro","name":"Granite 4.0 Micro","modelName":"Granite 4.0 Micro","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":108.4,"scoreLo":102.04,"scoreHi":113.92,"reasoningOffset":0.0,"rankEligible":true,"rankMean":640.8953,"rankLo":592.0,"rankHi":688.0},{"stateId":"granite-4-1-30b::default","configSlug":"granite-4-1-30b","modelId":"granite-4-1-30b","modelSlug":"granite-4-1-30b","name":"Granite 4.1 30B","modelName":"Granite 4.1 30B","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":124.67,"scoreLo":117.44,"scoreHi":130.24,"reasoningOffset":0.0,"rankEligible":true,"rankMean":440.7778,"rankLo":374.0,"rankHi":526.0},{"stateId":"granite-4-1-3b::default","configSlug":"granite-4-1-3b","modelId":"granite-4-1-3b","modelSlug":"granite-4-1-3b","name":"Granite 4.1 3B","modelName":"Granite 4.1 3B","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":116.66,"scoreLo":108.49,"scoreHi":123.51,"reasoningOffset":0.0,"rankEligible":true,"rankMean":542.4922,"rankLo":460.0,"rankHi":632.0},{"stateId":"granite-4-1-8b::default","configSlug":"granite-4-1-8b","modelId":"granite-4-1-8b","modelSlug":"granite-4-1-8b","name":"Granite 4.1 8B","modelName":"Granite 4.1 8B","organizationSlug":"ibm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":126.54,"scoreLo":122.02,"scoreHi":130.55,"reasoningOffset":0.0,"rankEligible":true,"rankMean":417.2933,"rankLo":374.0,"rankHi":467.0},{"stateId":"granite-4-2-30b::default","configSlug":"granite-4-2-30b","modelId":"granite-4-2-30b","modelSlug":"granite-4-2-30b","name":"Granite 4.2 30B","modelName":"Granite 4.2 30B","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":1,"score":138.15,"scoreLo":133.86,"scoreHi":142.46,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"granite-4-2-30b::enabled","configSlug":"granite-4-2-30b-thinking","modelId":"granite-4-2-30b","modelSlug":"granite-4-2-30b","name":"Granite 4.2 30B (thinking)","modelName":"Granite 4.2 30B","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":139.81,"scoreLo":136.42,"scoreHi":143.03,"reasoningOffset":1.66,"rankEligible":true,"rankMean":243.0453,"rankLo":209.0,"rankHi":284.025},{"stateId":"granite-4-2-3b::default","configSlug":"granite-4-2-3b","modelId":"granite-4-2-3b","modelSlug":"granite-4-2-3b","name":"Granite 4.2 3B","modelName":"Granite 4.2 3B","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":1,"score":127.74,"scoreLo":122.2,"scoreHi":132.71,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"granite-4-2-3b::enabled","configSlug":"granite-4-2-3b-thinking","modelId":"granite-4-2-3b","modelSlug":"granite-4-2-3b","name":"Granite 4.2 3B (thinking)","modelName":"Granite 4.2 3B","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":128.63,"scoreLo":123.86,"scoreHi":132.92,"reasoningOffset":0.89,"rankEligible":true,"rankMean":391.7586,"rankLo":340.0,"rankHi":444.0},{"stateId":"granite-4-2-8b::default","configSlug":"granite-4-2-8b","modelId":"granite-4-2-8b","modelSlug":"granite-4-2-8b","name":"Granite 4.2 8B","modelName":"Granite 4.2 8B","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":133.3,"scoreLo":128.96,"scoreHi":137.37,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"granite-4-2-8b::enabled","configSlug":"granite-4-2-8b-thinking","modelId":"granite-4-2-8b","modelSlug":"granite-4-2-8b","name":"Granite 4.2 8B (thinking)","modelName":"Granite 4.2 8B","organizationSlug":"ibm","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":135.26,"scoreLo":131.84,"scoreHi":138.72,"reasoningOffset":1.97,"rankEligible":true,"rankMean":304.5317,"rankLo":262.0,"rankHi":348.0},{"stateId":"grok-1-2023-11::default","configSlug":"grok-1-2023-11","modelId":"grok-1-2023-11","modelSlug":"grok-1-2023-11","name":"Grok 1 (November 2023)","modelName":"Grok 1 (November 2023)","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":105.68,"scoreLo":96.86,"scoreHi":113.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":667.3894,"rankLo":590.975,"rankHi":727.0},{"stateId":"grok-1-5::default","configSlug":"grok-1-5","modelId":"grok-1-5","modelSlug":"grok-1-5","name":"Grok 1.5","modelName":"Grok 1.5","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":115.66,"scoreLo":109.75,"scoreHi":120.93,"reasoningOffset":0.0,"rankEligible":true,"rankMean":554.3642,"rankLo":501.0,"rankHi":610.0},{"stateId":"grok-1-5v::default","configSlug":"grok-1-5v","modelId":"grok-1-5v","modelSlug":"grok-1-5v","name":"Grok 1.5V","modelName":"Grok 1.5V","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":113.57,"scoreLo":105.99,"scoreHi":120.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":579.4831,"rankLo":505.0,"rankHi":656.025},{"stateId":"grok-2::default","configSlug":"grok-2","modelId":"grok-2","modelSlug":"grok-2","name":"Grok 2","modelName":"Grok 2","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":129.71,"scoreLo":125.92,"scoreHi":133.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":378.3906,"rankLo":333.975,"rankHi":420.0},{"stateId":"grok-2-1212::default","configSlug":"grok-2-1212","modelId":"grok-2-1212","modelSlug":"grok-2-1212","name":"Grok 2 1212","modelName":"Grok 2 1212","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":127.64,"scoreLo":124.37,"scoreHi":130.7,"reasoningOffset":0.0,"rankEligible":true,"rankMean":404.2883,"rankLo":369.0,"rankHi":436.0},{"stateId":"grok-2-mini::default","configSlug":"grok-2-mini","modelId":"grok-2-mini","modelSlug":"grok-2-mini","name":"Grok 2 Mini","modelName":"Grok 2 Mini","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":127.0,"scoreLo":122.74,"scoreHi":130.94,"reasoningOffset":0.0,"rankEligible":true,"rankMean":411.7264,"rankLo":365.975,"rankHi":456.0},{"stateId":"grok-2-vision-1212::default","configSlug":"grok-2-vision-1212","modelId":"grok-2-vision-1212","modelSlug":"grok-2-vision-1212","name":"Grok 2 Vision 1212","modelName":"Grok 2 Vision 1212","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":126.4,"scoreLo":118.57,"scoreHi":134.27,"reasoningOffset":0.0,"rankEligible":true,"rankMean":419.3381,"rankLo":321.0,"rankHi":517.0},{"stateId":"grok-3::default","configSlug":"grok-3","modelId":"grok-3","modelSlug":"grok-3","name":"Grok 3 (API)","modelName":"Grok 3 (API)","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":"none","reasoningCondition":"default","isDefault":true,"n":30,"score":137.18,"scoreLo":134.76,"scoreHi":139.59,"reasoningOffset":0.0,"rankEligible":true,"rankMean":278.2706,"rankLo":252.0,"rankHi":306.0},{"stateId":"grok-3-mini::default","configSlug":"grok-3-mini","modelId":"grok-3-mini","modelSlug":"grok-3-mini","name":"Grok 3 Mini","modelName":"Grok 3 Mini","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":139.58,"scoreLo":136.27,"scoreHi":142.77,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"grok-3-mini::enabled","configSlug":"grok-3-mini-beta","modelId":"grok-3-mini","modelSlug":"grok-3-mini","name":"Grok 3 Mini (enabled)","modelName":"Grok 3 Mini","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":140.33,"scoreLo":135.87,"scoreHi":144.76,"reasoningOffset":0.75,"rankEligible":false},{"stateId":"grok-3-mini::effort:high","configSlug":"grok-3-mini-high","modelId":"grok-3-mini","modelSlug":"grok-3-mini","name":"Grok 3 Mini (high)","modelName":"Grok 3 Mini","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":26,"score":140.81,"scoreLo":138.27,"scoreHi":143.28,"reasoningOffset":1.24,"rankEligible":true,"rankMean":229.6803,"rankLo":209.0,"rankHi":257.0},{"stateId":"grok-3-mini::effort:low","configSlug":"grok-3-mini-low","modelId":"grok-3-mini","modelSlug":"grok-3-mini","name":"Grok 3 Mini (low)","modelName":"Grok 3 Mini","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":15,"score":138.44,"scoreLo":136.08,"scoreHi":140.86,"reasoningOffset":-1.14,"rankEligible":true},{"stateId":"grok-4-0709::default","configSlug":"grok-4-0709","modelId":"grok-4-0709","modelSlug":"grok-4-0709","name":"Grok 4","modelName":"Grok 4","organizationSlug":"xai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":97,"score":146.06,"scoreLo":144.13,"scoreHi":148.34,"reasoningOffset":2.4,"rankEligible":true,"rankMean":174.6767,"rankLo":164.0,"rankHi":186.0},{"stateId":"grok-4-0709::effort:high","configSlug":"grok-4-0709-high-aider-polyglot-diff","modelId":"grok-4-0709","modelSlug":"grok-4-0709","name":"Grok 4 (high)","modelName":"Grok 4","organizationSlug":"xai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":144.52,"scoreLo":141.48,"scoreHi":147.82,"reasoningOffset":0.86,"rankEligible":false},{"stateId":"grok-4-0709::effort:minimal","configSlug":"grok-4-0709-minimal","modelId":"grok-4-0709","modelSlug":"grok-4-0709","name":"Grok 4 (minimal)","modelName":"Grok 4","organizationSlug":"xai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":139.62,"scoreLo":134.96,"scoreHi":144.66,"reasoningOffset":-4.04,"rankEligible":false},{"stateId":"grok-4-1::default","configSlug":"grok-4-1","modelId":"grok-4-1","modelSlug":"grok-4-1","name":"Grok 4.1","modelName":"Grok 4.1","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":1,"score":143.13,"scoreLo":138.53,"scoreHi":148.05,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"grok-4-1-fast::default","configSlug":"grok-4-1-fast","modelId":"grok-4-1-fast","modelSlug":"grok-4-1-fast","name":"Grok 4.1 Fast","modelName":"Grok 4.1 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":17,"score":142.78,"scoreLo":140.61,"scoreHi":145.04,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"grok-4-1-fast::effort:high","configSlug":"grok-4-1-fast-high","modelId":"grok-4-1-fast","modelSlug":"grok-4-1-fast","name":"Grok 4.1 Fast (high)","modelName":"Grok 4.1 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":144.33,"scoreLo":141.43,"scoreHi":147.42,"reasoningOffset":1.55,"rankEligible":false},{"stateId":"grok-4-1-fast::disabled","configSlug":"grok-4-1-fast-no-thinking","modelId":"grok-4-1-fast","modelSlug":"grok-4-1-fast","name":"Grok 4.1 Fast (disabled)","modelName":"Grok 4.1 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":30,"score":135.74,"scoreLo":133.44,"scoreHi":137.92,"reasoningOffset":-7.04,"rankEligible":true},{"stateId":"grok-4-1-fast::enabled","configSlug":"grok-4-1-fast-thinking","modelId":"grok-4-1-fast","modelSlug":"grok-4-1-fast","name":"Grok 4.1 Fast (thinking)","modelName":"Grok 4.1 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":83,"score":144.71,"scoreLo":142.77,"scoreHi":146.87,"reasoningOffset":1.93,"rankEligible":true,"rankMean":189.0908,"rankLo":178.0,"rankHi":201.0},{"stateId":"grok-4-1::enabled","configSlug":"grok-4-1-thinking","modelId":"grok-4-1","modelSlug":"grok-4-1","name":"Grok 4.1 (thinking)","modelName":"Grok 4.1","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":4,"score":144.42,"scoreLo":139.75,"scoreHi":149.24,"reasoningOffset":1.29,"rankEligible":true,"rankMean":191.7581,"rankLo":144.0,"rankHi":242.0},{"stateId":"grok-4-20::default","configSlug":"grok-4-20","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":25,"score":147.85,"scoreLo":145.7,"scoreHi":150.36,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"grok-4-20::effort:high","configSlug":"grok-4-20-high","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20 (high)","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":4,"score":148.56,"scoreLo":145.65,"scoreHi":151.71,"reasoningOffset":0.71,"rankEligible":true},{"stateId":"grok-4-20::effort:low","configSlug":"grok-4-20-low","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20 (low)","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":147.35,"scoreLo":143.7,"scoreHi":151.49,"reasoningOffset":-0.5,"rankEligible":false},{"stateId":"grok-4-20::effort:medium","configSlug":"grok-4-20-medium","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20 (medium)","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":3,"score":146.64,"scoreLo":143.42,"scoreHi":150.28,"reasoningOffset":-1.21,"rankEligible":false},{"stateId":"grok-4-20::effort:minimal","configSlug":"grok-4-20-minimal","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20 (minimal)","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":143.78,"scoreLo":139.02,"scoreHi":148.65,"reasoningOffset":-4.07,"rankEligible":false},{"stateId":"grok-4-20::disabled","configSlug":"grok-4-20-no-thinking-0309","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20 (disabled)","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":23,"score":140.05,"scoreLo":137.64,"scoreHi":142.49,"reasoningOffset":-7.81,"rankEligible":true},{"stateId":"grok-4-20::enabled","configSlug":"grok-4-20-thinking","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20 (thinking)","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":72,"score":150.26,"scoreLo":148.01,"scoreHi":152.85,"reasoningOffset":2.4,"rankEligible":true,"rankMean":128.9317,"rankLo":118.0,"rankHi":140.0},{"stateId":"grok-4-20::effort:xhigh","configSlug":"grok-4-20-xhigh","modelId":"grok-4-20","modelSlug":"grok-4-20","name":"Grok 4.20 (xhigh)","modelName":"Grok 4.20","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":148.78,"scoreLo":145.8,"scoreHi":152.28,"reasoningOffset":0.93,"rankEligible":false},{"stateId":"grok-4-3::default","configSlug":"grok-4-3","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":45,"score":147.88,"scoreLo":145.78,"scoreHi":150.4,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"grok-4-3::effort:high","configSlug":"grok-4-3-high","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3 (high)","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":66,"score":148.99,"scoreLo":146.71,"scoreHi":151.56,"reasoningOffset":1.11,"rankEligible":true},{"stateId":"grok-4-3::effort:low","configSlug":"grok-4-3-low","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3 (low)","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":18,"score":147.44,"scoreLo":144.89,"scoreHi":150.25,"reasoningOffset":-0.44,"rankEligible":true},{"stateId":"grok-4-3::effort:medium","configSlug":"grok-4-3-medium","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3 (medium)","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":9,"score":149.55,"scoreLo":146.7,"scoreHi":152.85,"reasoningOffset":1.67,"rankEligible":true,"rankMean":136.6103,"rankLo":114.0,"rankHi":160.025},{"stateId":"grok-4-3::effort:minimal","configSlug":"grok-4-3-minimal","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3 (minimal)","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":143.81,"scoreLo":139.29,"scoreHi":148.79,"reasoningOffset":-4.07,"rankEligible":false},{"stateId":"grok-4-3::disabled","configSlug":"grok-4-3-no-thinking","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3 (disabled)","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":19,"score":140.84,"scoreLo":138.26,"scoreHi":143.43,"reasoningOffset":-7.04,"rankEligible":true},{"stateId":"grok-4-3::enabled","configSlug":"grok-4-3-thinking-ale-bench-self-refine-x1","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3 (thinking)","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":149.01,"scoreLo":145.43,"scoreHi":152.94,"reasoningOffset":1.13,"rankEligible":false},{"stateId":"grok-4-3::effort:xhigh","configSlug":"grok-4-3-xhigh","modelId":"grok-4-3","modelSlug":"grok-4-3","name":"Grok 4.3 (xhigh)","modelName":"Grok 4.3","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":3,"score":149.04,"scoreLo":146.19,"scoreHi":152.12,"reasoningOffset":1.16,"rankEligible":false},{"stateId":"grok-4-5::effort:high","configSlug":"grok-4-5","modelId":"grok-4-5","modelSlug":"grok-4-5","name":"Grok 4.5 (high)","modelName":"Grok 4.5","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":true,"n":133,"score":158.79,"scoreLo":155.72,"scoreHi":162.39,"reasoningOffset":-0.87,"rankEligible":true},{"stateId":"grok-4-5::effort:low","configSlug":"grok-4-5-low","modelId":"grok-4-5","modelSlug":"grok-4-5","name":"Grok 4.5 (low)","modelName":"Grok 4.5","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":12,"score":158.11,"scoreLo":154.89,"scoreHi":161.75,"reasoningOffset":-1.54,"rankEligible":true},{"stateId":"grok-4-5::effort:medium","configSlug":"grok-4-5-medium","modelId":"grok-4-5","modelSlug":"grok-4-5","name":"Grok 4.5 (medium)","modelName":"Grok 4.5","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":16,"score":159.65,"scoreLo":156.33,"scoreHi":163.61,"reasoningOffset":-0.01,"rankEligible":true},{"stateId":"grok-4-5::effort:minimal","configSlug":"grok-4-5-minimal","modelId":"grok-4-5","modelSlug":"grok-4-5","name":"Grok 4.5 (minimal)","modelName":"Grok 4.5","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":155.11,"scoreLo":150.23,"scoreHi":160.47,"reasoningOffset":-4.54,"rankEligible":false},{"stateId":"grok-4-5::enabled","configSlug":"grok-4-5-pi","modelId":"grok-4-5","modelSlug":"grok-4-5","name":"Grok 4.5 (enabled)","modelName":"Grok 4.5","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":11,"score":159.62,"scoreLo":156.31,"scoreHi":163.66,"reasoningOffset":-0.03,"rankEligible":true},{"stateId":"grok-4-5::default","configSlug":"grok-4-5-unknown","modelId":"grok-4-5","modelSlug":"grok-4-5","name":"Grok 4.5","modelName":"Grok 4.5","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":36,"score":159.66,"scoreLo":156.48,"scoreHi":163.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":52.4917,"rankLo":45.0,"rankHi":61.0},{"stateId":"grok-4-6::default","configSlug":"grok-4-6","modelId":"grok-4-6","modelSlug":"grok-4-6","name":"Grok 4.6","modelName":"Grok 4.6","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":62,"score":161.77,"scoreLo":158.34,"scoreHi":165.83,"reasoningOffset":0.0,"rankEligible":true,"rankMean":29.9567,"rankLo":24.0,"rankHi":37.0},{"stateId":"grok-4-6::enabled","configSlug":"grok-4-6-epoch-automation","modelId":"grok-4-6","modelSlug":"grok-4-6","name":"Grok 4.6 (enabled)","modelName":"Grok 4.6","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":162.0,"scoreLo":157.99,"scoreHi":166.71,"reasoningOffset":0.23,"rankEligible":false},{"stateId":"grok-4-6::effort:high","configSlug":"grok-4-6-high","modelId":"grok-4-6","modelSlug":"grok-4-6","name":"Grok 4.6 (high)","modelName":"Grok 4.6","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":88,"score":160.95,"scoreLo":157.66,"scoreHi":164.88,"reasoningOffset":-0.82,"rankEligible":true},{"stateId":"grok-4-6::effort:low","configSlug":"grok-4-6-low","modelId":"grok-4-6","modelSlug":"grok-4-6","name":"Grok 4.6 (low)","modelName":"Grok 4.6","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":22,"score":157.87,"scoreLo":154.82,"scoreHi":161.56,"reasoningOffset":-3.9,"rankEligible":true},{"stateId":"grok-4-6::effort:medium","configSlug":"grok-4-6-medium","modelId":"grok-4-6","modelSlug":"grok-4-6","name":"Grok 4.6 (medium)","modelName":"Grok 4.6","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":25,"score":160.67,"scoreLo":157.29,"scoreHi":164.68,"reasoningOffset":-1.11,"rankEligible":true},{"stateId":"grok-4-6::effort:xhigh","configSlug":"grok-4-6-xhigh","modelId":"grok-4-6","modelSlug":"grok-4-6","name":"Grok 4.6 (xhigh)","modelName":"Grok 4.6","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":81,"score":161.22,"scoreLo":157.94,"scoreHi":165.17,"reasoningOffset":-0.55,"rankEligible":true},{"stateId":"grok-4-7::default","configSlug":"grok-4-7","modelId":"grok-4-7","modelSlug":"grok-4-7","name":"Grok 4.7","modelName":"Grok 4.7","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":44,"score":162.43,"scoreLo":158.84,"scoreHi":166.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":24.3531,"rankLo":20.0,"rankHi":31.0},{"stateId":"grok-4-7::enabled","configSlug":"grok-4-7-concept-synth-cascade","modelId":"grok-4-7","modelSlug":"grok-4-7","name":"Grok 4.7 (enabled)","modelName":"Grok 4.7","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":162.6,"scoreLo":158.66,"scoreHi":167.14,"reasoningOffset":0.18,"rankEligible":false},{"stateId":"grok-4-7::effort:high","configSlug":"grok-4-7-high","modelId":"grok-4-7","modelSlug":"grok-4-7","name":"Grok 4.7 (high)","modelName":"Grok 4.7","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":36,"score":161.79,"scoreLo":158.37,"scoreHi":166.0,"reasoningOffset":-0.64,"rankEligible":true},{"stateId":"grok-4-7::effort:low","configSlug":"grok-4-7-low","modelId":"grok-4-7","modelSlug":"grok-4-7","name":"Grok 4.7 (low)","modelName":"Grok 4.7","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":20,"score":158.75,"scoreLo":155.59,"scoreHi":162.64,"reasoningOffset":-3.68,"rankEligible":true},{"stateId":"grok-4-7::effort:medium","configSlug":"grok-4-7-medium","modelId":"grok-4-7","modelSlug":"grok-4-7","name":"Grok 4.7 (medium)","modelName":"Grok 4.7","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":13,"score":160.85,"scoreLo":157.41,"scoreHi":164.84,"reasoningOffset":-1.57,"rankEligible":true},{"stateId":"grok-4-7::effort:xhigh","configSlug":"grok-4-7-xhigh","modelId":"grok-4-7","modelSlug":"grok-4-7","name":"Grok 4.7 (xhigh)","modelName":"Grok 4.7","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":122,"score":161.31,"scoreLo":157.96,"scoreHi":165.3,"reasoningOffset":-1.12,"rankEligible":true},{"stateId":"grok-4-fast::default","configSlug":"grok-4-fast","modelId":"grok-4-fast","modelSlug":"grok-4-fast","name":"Grok 4 Fast","modelName":"Grok 4 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":8,"score":141.7,"scoreLo":139.17,"scoreHi":144.35,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"grok-4-fast::effort:high","configSlug":"grok-4-fast-high","modelId":"grok-4-fast","modelSlug":"grok-4-fast","name":"Grok 4 Fast (high)","modelName":"Grok 4 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":142.11,"scoreLo":138.48,"scoreHi":145.79,"reasoningOffset":0.42,"rankEligible":false},{"stateId":"grok-4-fast::effort:minimal","configSlug":"grok-4-fast-minimal","modelId":"grok-4-fast","modelSlug":"grok-4-fast","name":"Grok 4 Fast (minimal)","modelName":"Grok 4 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":137.64,"scoreLo":132.68,"scoreHi":142.89,"reasoningOffset":-4.06,"rankEligible":false},{"stateId":"grok-4-fast::disabled","configSlug":"grok-4-fast-no-thinking","modelId":"grok-4-fast","modelSlug":"grok-4-fast","name":"Grok 4 Fast (disabled)","modelName":"Grok 4 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":23,"score":134.09,"scoreLo":131.51,"scoreHi":136.52,"reasoningOffset":-7.61,"rankEligible":true},{"stateId":"grok-4-fast::enabled","configSlug":"grok-4-fast-thinking","modelId":"grok-4-fast","modelSlug":"grok-4-fast","name":"Grok 4 Fast (thinking)","modelName":"Grok 4 Fast","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":63,"score":143.77,"scoreLo":141.72,"scoreHi":145.99,"reasoningOffset":2.08,"rankEligible":true,"rankMean":199.0608,"rankLo":185.0,"rankHi":212.0},{"stateId":"grok-beta::default","configSlug":"grok-beta","modelId":"grok-beta","modelSlug":"grok-beta","name":"Grok Beta","modelName":"Grok Beta","organizationSlug":"xai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":126.94,"scoreLo":120.82,"scoreHi":132.81,"reasoningOffset":0.0,"rankEligible":true,"rankMean":412.4953,"rankLo":340.0,"rankHi":483.025},{"stateId":"grok-build-0-1::default","configSlug":"grok-build-0-1","modelId":"grok-build-0-1","modelSlug":"grok-build-0-1","name":"Grok Build 0.1","modelName":"Grok Build 0.1","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":150.84,"scoreLo":147.81,"scoreHi":154.02,"reasoningOffset":0.0,"rankEligible":true,"rankMean":122.7228,"rankLo":104.0,"rankHi":146.0},{"stateId":"grok-build-0-1::effort:high","configSlug":"grok-build-0-1-high","modelId":"grok-build-0-1","modelSlug":"grok-build-0-1","name":"Grok Build 0.1 (high)","modelName":"Grok Build 0.1","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":151.39,"scoreLo":148.09,"scoreHi":154.98,"reasoningOffset":0.55,"rankEligible":false},{"stateId":"grok-build-0-1::effort:low","configSlug":"grok-build-0-1-low","modelId":"grok-build-0-1","modelSlug":"grok-build-0-1","name":"Grok Build 0.1 (low)","modelName":"Grok Build 0.1","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":150.0,"scoreLo":145.86,"scoreHi":154.53,"reasoningOffset":-0.84,"rankEligible":false},{"stateId":"grok-build-0-1::effort:medium","configSlug":"grok-build-0-1-medium","modelId":"grok-build-0-1","modelSlug":"grok-build-0-1","name":"Grok Build 0.1 (medium)","modelName":"Grok Build 0.1","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":151.43,"scoreLo":147.25,"scoreHi":156.16,"reasoningOffset":0.59,"rankEligible":false},{"stateId":"grok-build-0-1::effort:xhigh","configSlug":"grok-build-0-1-xhigh","modelId":"grok-build-0-1","modelSlug":"grok-build-0-1","name":"Grok Build 0.1 (xhigh)","modelName":"Grok Build 0.1","organizationSlug":"xai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":151.66,"scoreLo":147.97,"scoreHi":155.83,"reasoningOffset":0.82,"rankEligible":false},{"stateId":"grok-code-fast-1::default","configSlug":"grok-code-fast-1","modelId":"grok-code-fast-1","modelSlug":"grok-code-fast-1","name":"Grok Code Fast 1","modelName":"Grok Code Fast 1","organizationSlug":"xai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":138.6,"scoreLo":135.91,"scoreHi":141.23,"reasoningOffset":2.4,"rankEligible":true,"rankMean":258.8942,"rankLo":228.0,"rankHi":292.0},{"stateId":"hermes-3-llama-3-1-405b::default","configSlug":"hermes-3-llama-3-1-405b","modelId":"hermes-3-llama-3-1-405b","modelSlug":"hermes-3-llama-3-1-405b","name":"Hermes 3 Llama 3.1 405B","modelName":"Hermes 3 Llama 3.1 405B","organizationSlug":"nous-research","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":124.39,"scoreLo":118.3,"scoreHi":130.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":444.4597,"rankLo":367.0,"rankHi":518.0},{"stateId":"hermes-3-llama-3-1-70b::default","configSlug":"hermes-3-llama-3-1-70b","modelId":"hermes-3-llama-3-1-70b","modelSlug":"hermes-3-llama-3-1-70b","name":"Hermes 3 Llama 3.1 70B","modelName":"Hermes 3 Llama 3.1 70B","organizationSlug":"nous-research","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":117.73,"scoreLo":113.2,"scoreHi":121.85,"reasoningOffset":0.0,"rankEligible":true,"rankMean":530.2006,"rankLo":489.0,"rankHi":570.0},{"stateId":"hermes-4-405b::default","configSlug":"hermes-4-405b","modelId":"hermes-4-405b","modelSlug":"hermes-4-405b","name":"Hermes 4 405B","modelName":"Hermes 4 405B","organizationSlug":"nous-research","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":135.8,"scoreLo":131.76,"scoreHi":139.75,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"hermes-4-70b::default","configSlug":"hermes-4-70b","modelId":"hermes-4-70b","modelSlug":"hermes-4-70b","name":"Hermes 4 70B","modelName":"Hermes 4 70B","organizationSlug":"nous-research","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":126.51,"scoreLo":121.44,"scoreHi":131.14,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"hermes-4-70b::disabled","configSlug":"hermes-4-70b-no-thinking","modelId":"hermes-4-70b","modelSlug":"hermes-4-70b","name":"Hermes 4 70B (disabled)","modelName":"Hermes 4 70B","organizationSlug":"nous-research","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":122.15,"scoreLo":117.51,"scoreHi":126.39,"reasoningOffset":-4.36,"rankEligible":true},{"stateId":"hermes-4-70b::enabled","configSlug":"hermes-4-70b-thinking","modelId":"hermes-4-70b","modelSlug":"hermes-4-70b","name":"Hermes 4 70B (thinking)","modelName":"Hermes 4 70B","organizationSlug":"nous-research","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":128.51,"scoreLo":124.32,"scoreHi":132.45,"reasoningOffset":1.99,"rankEligible":true,"rankMean":393.4169,"rankLo":347.0,"rankHi":438.0},{"stateId":"hermes-4-405b::disabled","configSlug":"hermes-4-llama-3-1-405b-no-thinking","modelId":"hermes-4-405b","modelSlug":"hermes-4-405b","name":"Hermes 4 405B (disabled)","modelName":"Hermes 4 405B","organizationSlug":"nous-research","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":132.49,"scoreLo":128.82,"scoreHi":135.98,"reasoningOffset":-3.32,"rankEligible":true},{"stateId":"hermes-4-405b::enabled","configSlug":"hermes-4-llama-3-1-405b-thinking","modelId":"hermes-4-405b","modelSlug":"hermes-4-405b","name":"Hermes 4 405B (thinking)","modelName":"Hermes 4 405B","organizationSlug":"nous-research","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":136.41,"scoreLo":133.03,"scoreHi":139.83,"reasoningOffset":0.61,"rankEligible":true,"rankMean":288.8006,"rankLo":246.0,"rankHi":333.0},{"stateId":"hy3::default","configSlug":"hy3","modelId":"hy3","modelSlug":"hy3","name":"Hy3","modelName":"Hy3","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":153.0,"scoreLo":150.26,"scoreHi":156.2,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"hy3::effort:high","configSlug":"hy3-high","modelId":"hy3","modelSlug":"hy3","name":"Hy3 (high)","modelName":"Hy3","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":12,"score":153.7,"scoreLo":150.52,"scoreHi":157.34,"reasoningOffset":0.7,"rankEligible":true,"rankMean":95.8572,"rankLo":80.0,"rankHi":113.0},{"stateId":"hy3::effort:low","configSlug":"hy3-low","modelId":"hy3","modelSlug":"hy3","name":"Hy3 (low)","modelName":"Hy3","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":3,"score":152.08,"scoreLo":148.01,"scoreHi":156.72,"reasoningOffset":-0.92,"rankEligible":false},{"stateId":"hy3::disabled","configSlug":"hy3-no-thinking","modelId":"hy3","modelSlug":"hy3","name":"Hy3 (disabled)","modelName":"Hy3","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":3,"score":145.96,"scoreLo":142.16,"scoreHi":149.94,"reasoningOffset":-7.04,"rankEligible":false},{"stateId":"hy3-preview::default","configSlug":"hy3-preview","modelId":"hy3-preview","modelSlug":"hy3-preview","name":"Hy3 preview","modelName":"Hy3 preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":7,"score":149.14,"scoreLo":146.21,"scoreHi":152.38,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"hy3-preview::effort:high","configSlug":"hy3-preview-high","modelId":"hy3-preview","modelSlug":"hy3-preview","name":"Hy3 preview (high)","modelName":"Hy3 preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":4,"score":150.14,"scoreLo":146.96,"scoreHi":153.67,"reasoningOffset":1.0,"rankEligible":true,"rankMean":130.2489,"rankLo":106.0,"rankHi":158.0},{"stateId":"hy3-preview::effort:low","configSlug":"hy3-preview-low","modelId":"hy3-preview","modelSlug":"hy3-preview","name":"Hy3 preview (low)","modelName":"Hy3 preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":147.71,"scoreLo":143.28,"scoreHi":152.29,"reasoningOffset":-1.43,"rankEligible":false},{"stateId":"hy3-preview::disabled","configSlug":"hy3-preview-no-thinking","modelId":"hy3-preview","modelSlug":"hy3-preview","name":"Hy3 preview (disabled)","modelName":"Hy3 preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":143.45,"scoreLo":139.73,"scoreHi":147.13,"reasoningOffset":-5.69,"rankEligible":true},{"stateId":"hy3-preview::enabled","configSlug":"hy3-preview-thinking","modelId":"hy3-preview","modelSlug":"hy3-preview","name":"Hy3 preview (thinking)","modelName":"Hy3 preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":150.14,"scoreLo":146.69,"scoreHi":153.86,"reasoningOffset":1.0,"rankEligible":true},{"stateId":"hy3-preview::effort:xhigh","configSlug":"hy3-preview-xhigh-free-genebench-tools","modelId":"hy3-preview","modelSlug":"hy3-preview","name":"Hy3 preview (xhigh)","modelName":"Hy3 preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":149.87,"scoreLo":146.22,"scoreHi":153.85,"reasoningOffset":0.73,"rankEligible":false},{"stateId":"hy4-preview::default","configSlug":"hy4-preview","modelId":"hy4-preview","modelSlug":"hy4-preview","name":"Hy4 Preview","modelName":"Hy4 Preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":34,"score":159.54,"scoreLo":156.26,"scoreHi":163.4,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"hy4-preview::effort:high","configSlug":"hy4-preview-high","modelId":"hy4-preview","modelSlug":"hy4-preview","name":"Hy4 Preview (high)","modelName":"Hy4 Preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":5,"score":160.09,"scoreLo":156.24,"scoreHi":164.59,"reasoningOffset":0.55,"rankEligible":true,"rankMean":47.9661,"rankLo":26.0,"rankHi":69.0},{"stateId":"hy4-preview::disabled","configSlug":"hy4-preview-no-thinking","modelId":"hy4-preview","modelSlug":"hy4-preview","name":"Hy4 Preview (disabled)","modelName":"Hy4 Preview","organizationSlug":"tencent","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":154.7,"scoreLo":147.75,"scoreHi":162.2,"reasoningOffset":-4.84,"rankEligible":false},{"stateId":"hyperclovax-seed-think-32b::default","configSlug":"hyperclovax-seed-think-32b","modelId":"hyperclovax-seed-think-32b","modelSlug":"hyperclovax-seed-think-32b","name":"HyperCLOVA X SEED 32B Think","modelName":"HyperCLOVA X SEED 32B Think","organizationSlug":"naver","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":8,"score":133.82,"scoreLo":129.37,"scoreHi":138.15,"reasoningOffset":0.0,"rankEligible":true,"rankMean":323.9525,"rankLo":268.0,"rankHi":381.0},{"stateId":"hypernova-60b-2605::effort:high","configSlug":"hypernova-60b-2605","modelId":"hypernova-60b-2605","modelSlug":"hypernova-60b-2605","name":"HyperNova 60B 2605 (high)","modelName":"HyperNova 60B 2605","organizationSlug":"multiverse-computing","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":true,"n":10,"score":138.76,"scoreLo":135.04,"scoreHi":142.35,"reasoningOffset":0.51,"rankEligible":true,"rankMean":256.8983,"rankLo":216.0,"rankHi":305.025},{"stateId":"inflection-1::default","configSlug":"inflection-1","modelId":"inflection-1","modelSlug":"inflection-1","name":"Inflection-1","modelName":"Inflection-1","organizationSlug":"inflection-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":102.56,"scoreLo":93.64,"scoreHi":110.5,"reasoningOffset":0.0,"rankEligible":true,"rankMean":696.4483,"rankLo":631.0,"rankHi":753.0},{"stateId":"inkling::default","configSlug":"inkling","modelId":"inkling","modelSlug":"inkling","name":"Inkling","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":30,"score":150.01,"scoreLo":147.69,"scoreHi":152.66,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"inkling::effort:0.99","configSlug":"inkling-effort-0-99","modelId":"inkling","modelSlug":"inkling","name":"Inkling (0.99)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"0.99","reasoningCondition":"effort:0.99","isDefault":false,"n":67,"score":151.37,"scoreLo":149.04,"scoreHi":154.14,"reasoningOffset":1.36,"rankEligible":true,"rankMean":116.6108,"rankLo":108.0,"rankHi":126.0},{"stateId":"inkling::effort:high","configSlug":"inkling-high","modelId":"inkling","modelSlug":"inkling","name":"Inkling (high)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":24,"score":150.79,"scoreLo":148.34,"scoreHi":153.59,"reasoningOffset":0.77,"rankEligible":true},{"stateId":"inkling::enabled","configSlug":"inkling-jobbench-opencode","modelId":"inkling","modelSlug":"inkling","name":"Inkling (enabled)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":151.62,"scoreLo":147.98,"scoreHi":155.62,"reasoningOffset":1.61,"rankEligible":false},{"stateId":"inkling::effort:low","configSlug":"inkling-low","modelId":"inkling","modelSlug":"inkling","name":"Inkling (low)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":148.18,"scoreLo":144.48,"scoreHi":152.42,"reasoningOffset":-1.83,"rankEligible":false},{"stateId":"inkling::effort:medium","configSlug":"inkling-medium","modelId":"inkling","modelSlug":"inkling","name":"Inkling (medium)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":5,"score":149.3,"scoreLo":145.96,"scoreHi":152.95,"reasoningOffset":-0.71,"rankEligible":true},{"stateId":"inkling::effort:minimal","configSlug":"inkling-minimal","modelId":"inkling","modelSlug":"inkling","name":"Inkling (minimal)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":2,"score":146.61,"scoreLo":142.97,"scoreHi":150.84,"reasoningOffset":-3.4,"rankEligible":false},{"stateId":"inkling::disabled","configSlug":"inkling-no-thinking","modelId":"inkling","modelSlug":"inkling","name":"Inkling (disabled)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":2,"score":143.62,"scoreLo":138.82,"scoreHi":149.16,"reasoningOffset":-6.39,"rankEligible":false},{"stateId":"inkling-small::default","configSlug":"inkling-small","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":26,"score":150.15,"scoreLo":147.87,"scoreHi":152.85,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"inkling-small::effort:0.99","configSlug":"inkling-small-effort-0-99","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (0.99)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"0.99","reasoningCondition":"effort:0.99","isDefault":false,"n":59,"score":151.84,"scoreLo":149.42,"scoreHi":154.58,"reasoningOffset":1.69,"rankEligible":true,"rankMean":112.1944,"rankLo":104.0,"rankHi":121.0},{"stateId":"inkling-small::effort:high","configSlug":"inkling-small-high","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (high)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":4,"score":151.46,"scoreLo":148.89,"scoreHi":154.41,"reasoningOffset":1.31,"rankEligible":true},{"stateId":"inkling-small::effort:low","configSlug":"inkling-small-low","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (low)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":3,"score":148.66,"scoreLo":146.32,"scoreHi":151.33,"reasoningOffset":-1.48,"rankEligible":false},{"stateId":"inkling-small::effort:max","configSlug":"inkling-small-max-chisel","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (max)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":2,"score":150.71,"scoreLo":147.66,"scoreHi":154.17,"reasoningOffset":0.56,"rankEligible":false},{"stateId":"inkling-small::effort:medium","configSlug":"inkling-small-medium","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (medium)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":3,"score":150.24,"scoreLo":147.75,"scoreHi":153.14,"reasoningOffset":0.09,"rankEligible":false},{"stateId":"inkling-small::effort:minimal","configSlug":"inkling-small-minimal","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (minimal)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":3,"score":148.06,"scoreLo":145.68,"scoreHi":150.68,"reasoningOffset":-2.08,"rankEligible":false},{"stateId":"inkling-small::disabled","configSlug":"inkling-small-no-thinking","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (disabled)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":2,"score":142.01,"scoreLo":138.37,"scoreHi":145.16,"reasoningOffset":-8.13,"rankEligible":false},{"stateId":"inkling-small-preview::effort:0.99","configSlug":"inkling-small-preview-effort-0-99","modelId":"inkling-small-preview","modelSlug":"inkling-small-preview","name":"Inkling-Small Preview (0.99)","modelName":"Inkling-Small Preview","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"0.99","reasoningCondition":"effort:0.99","isDefault":false,"n":12,"score":151.68,"scoreLo":148.35,"scoreHi":155.4,"reasoningOffset":1.32,"rankEligible":true,"rankMean":114.3714,"rankLo":92.0,"rankHi":139.0},{"stateId":"inkling-small::enabled","configSlug":"inkling-small-wordlebench-high","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (enabled)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":151.5,"scoreLo":147.91,"scoreHi":155.52,"reasoningOffset":1.36,"rankEligible":false},{"stateId":"inkling-small::effort:xhigh","configSlug":"inkling-small-xhigh","modelId":"inkling-small","modelSlug":"inkling-small","name":"Inkling-Small (xhigh)","modelName":"Inkling-Small","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":152.41,"scoreLo":149.75,"scoreHi":155.47,"reasoningOffset":2.26,"rankEligible":false},{"stateId":"inkling::effort:xhigh","configSlug":"inkling-xhigh","modelId":"inkling","modelSlug":"inkling","name":"Inkling (xhigh)","modelName":"Inkling","organizationSlug":"thinking-machines-lab","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":9,"score":150.66,"scoreLo":148.15,"scoreHi":153.52,"reasoningOffset":0.64,"rankEligible":true},{"stateId":"intellect-1::default","configSlug":"intellect-1","modelId":"intellect-1","modelSlug":"intellect-1","name":"INTELLECT-1","modelName":"INTELLECT-1","organizationSlug":"prime-intellect","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":62.87,"scoreLo":46.19,"scoreHi":76.84,"reasoningOffset":0.0,"rankEligible":true,"rankMean":868.6717,"rankLo":854.0,"rankHi":879.025},{"stateId":"intellect-3::default","configSlug":"intellect-3","modelId":"intellect-3","modelSlug":"intellect-3","name":"INTELLECT-3","modelName":"INTELLECT-3","organizationSlug":"prime-intellect","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":137.97,"scoreLo":134.46,"scoreHi":141.47,"reasoningOffset":2.4,"rankEligible":true,"rankMean":267.5264,"rankLo":223.0,"rankHi":313.0},{"stateId":"internlm-20b::default","configSlug":"internlm-20b","modelId":"internlm-20b","modelSlug":"internlm-20b","name":"InternLM 20B","modelName":"InternLM 20B","organizationSlug":"shanghai-ai-lab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":97.1,"scoreLo":90.07,"scoreHi":103.12,"reasoningOffset":0.0,"rankEligible":true,"rankMean":741.64,"rankLo":714.0,"rankHi":768.0},{"stateId":"internlm-7b::default","configSlug":"internlm-7b","modelId":"internlm-7b","modelSlug":"internlm-7b","name":"InternLM 7B","modelName":"InternLM 7B","organizationSlug":"shanghai-ai-lab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":81.63,"scoreLo":71.7,"scoreHi":90.39,"reasoningOffset":0.0,"rankEligible":true,"rankMean":827.2333,"rankLo":806.0,"rankHi":847.0},{"stateId":"internvl-chat-v1-5::default","configSlug":"internvl-chat-v1-5","modelId":"internvl-chat-v1-5","modelSlug":"internvl-chat-v1-5","name":"InternVL-Chat-V1-5","modelName":"InternVL-Chat-V1-5","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":108.14,"scoreLo":101.09,"scoreHi":114.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":643.3492,"rankLo":583.0,"rankHi":695.0},{"stateId":"internvl2-40b::default","configSlug":"internvl2-40b","modelId":"internvl2-40b","modelSlug":"internvl2-40b","name":"InternVL2-40B","modelName":"InternVL2-40B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":119.07,"scoreLo":110.95,"scoreHi":127.21,"reasoningOffset":0.0,"rankEligible":true,"rankMean":512.3017,"rankLo":412.0,"rankHi":605.025},{"stateId":"internvl2-5-78b::default","configSlug":"internvl2-5-78b","modelId":"internvl2-5-78b","modelSlug":"internvl2-5-78b","name":"InternVL2.5-78B","modelName":"InternVL2.5-78B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":129.67,"scoreLo":126.39,"scoreHi":132.67,"reasoningOffset":0.0,"rankEligible":true,"rankMean":379.1169,"rankLo":343.0,"rankHi":412.0},{"stateId":"internvl3-14b::default","configSlug":"internvl3-14b","modelId":"internvl3-14b","modelSlug":"internvl3-14b","name":"InternVL3-14B","modelName":"InternVL3-14B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":128.65,"scoreLo":123.73,"scoreHi":133.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":391.5078,"rankLo":334.0,"rankHi":446.0},{"stateId":"internvl3-1b::default","configSlug":"internvl3-1b","modelId":"internvl3-1b","modelSlug":"internvl3-1b","name":"InternVL3-1B","modelName":"InternVL3-1B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":101.22,"scoreLo":93.3,"scoreHi":107.96,"reasoningOffset":0.0,"rankEligible":true,"rankMean":708.4839,"rankLo":663.0,"rankHi":754.0},{"stateId":"internvl3-2b::default","configSlug":"internvl3-2b","modelId":"internvl3-2b","modelSlug":"internvl3-2b","name":"InternVL3-2B","modelName":"InternVL3-2B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":110.35,"scoreLo":105.03,"scoreHi":114.94,"reasoningOffset":0.0,"rankEligible":true,"rankMean":619.1408,"rankLo":576.975,"rankHi":658.025},{"stateId":"internvl3-38b::default","configSlug":"internvl3-38b","modelId":"internvl3-38b","modelSlug":"internvl3-38b","name":"InternVL3-38B","modelName":"InternVL3-38B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":131.59,"scoreLo":126.87,"scoreHi":136.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":353.6783,"rankLo":292.0,"rankHi":410.0},{"stateId":"internvl3-5-241b-a28b::default","configSlug":"internvl3-5-241b-a28b","modelId":"internvl3-5-241b-a28b","modelSlug":"internvl3-5-241b-a28b","name":"InternVL3.5-241B-A28B","modelName":"InternVL3.5-241B-A28B","organizationSlug":"opengvlab","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":139.99,"scoreLo":135.97,"scoreHi":144.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":241.3683,"rankLo":197.0,"rankHi":292.0},{"stateId":"internvl3-78b::default","configSlug":"internvl3-78b","modelId":"internvl3-78b","modelSlug":"internvl3-78b","name":"InternVL3-78B","modelName":"InternVL3-78B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":133.2,"scoreLo":128.64,"scoreHi":137.97,"reasoningOffset":0.0,"rankEligible":true,"rankMean":332.3853,"rankLo":270.0,"rankHi":390.0},{"stateId":"internvl3-8b::default","configSlug":"internvl3-8b","modelId":"internvl3-8b","modelSlug":"internvl3-8b","name":"InternVL3-8B","modelName":"InternVL3-8B","organizationSlug":"opengvlab","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":123.75,"scoreLo":118.45,"scoreHi":128.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":452.6347,"rankLo":397.0,"rankHi":515.0},{"stateId":"jamba-large-1-5::default","configSlug":"jamba-large-1-5","modelId":"jamba-large-1-5","modelSlug":"jamba-large-1-5","name":"Jamba 1.5 Large","modelName":"Jamba 1.5 Large","organizationSlug":"ai21-labs","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":118.06,"scoreLo":112.7,"scoreHi":123.18,"reasoningOffset":0.0,"rankEligible":true,"rankMean":525.86,"rankLo":467.0,"rankHi":578.0},{"stateId":"jamba-large-1-6::default","configSlug":"jamba-large-1-6","modelId":"jamba-large-1-6","modelSlug":"jamba-large-1-6","name":"Jamba Large 1.6","modelName":"Jamba Large 1.6","organizationSlug":"ai21-labs","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":113.81,"scoreLo":108.45,"scoreHi":118.54,"reasoningOffset":0.0,"rankEligible":true,"rankMean":576.7722,"rankLo":533.0,"rankHi":625.0},{"stateId":"jamba-large-1-7::default","configSlug":"jamba-large-1-7","modelId":"jamba-large-1-7","modelSlug":"jamba-large-1-7","name":"Jamba Large 1.7","modelName":"Jamba Large 1.7","organizationSlug":"ai21-labs","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":119.86,"scoreLo":114.18,"scoreHi":125.08,"reasoningOffset":0.0,"rankEligible":true,"rankMean":503.3861,"rankLo":441.0,"rankHi":561.0},{"stateId":"jamba-mini-1-5::default","configSlug":"jamba-mini-1-5","modelId":"jamba-mini-1-5","modelSlug":"jamba-mini-1-5","name":"Jamba 1.5 Mini","modelName":"Jamba 1.5 Mini","organizationSlug":"ai21-labs","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":104.65,"scoreLo":97.75,"scoreHi":110.73,"reasoningOffset":0.0,"rankEligible":true,"rankMean":678.8394,"rankLo":631.975,"rankHi":720.0},{"stateId":"jamba-mini-1-6::default","configSlug":"jamba-mini-1-6","modelId":"jamba-mini-1-6","modelSlug":"jamba-mini-1-6","name":"Jamba Mini 1.6","modelName":"Jamba Mini 1.6","organizationSlug":"ai21-labs","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":101.45,"scoreLo":94.12,"scoreHi":107.69,"reasoningOffset":0.0,"rankEligible":true,"rankMean":706.7056,"rankLo":668.0,"rankHi":746.0},{"stateId":"jamba-mini-1-7::default","configSlug":"jamba-mini-1-7","modelId":"jamba-mini-1-7","modelSlug":"jamba-mini-1-7","name":"Jamba Mini 1.7","modelName":"Jamba Mini 1.7","organizationSlug":"ai21-labs","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":106.05,"scoreLo":98.0,"scoreHi":113.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":664.3733,"rankLo":595.0,"rankHi":723.0},{"stateId":"jamba-reasoning-3b::default","configSlug":"jamba-reasoning-3b","modelId":"jamba-reasoning-3b","modelSlug":"jamba-reasoning-3b","name":"Jamba Reasoning 3B","modelName":"Jamba Reasoning 3B","organizationSlug":"ai21-labs","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":116.53,"scoreLo":110.16,"scoreHi":122.03,"reasoningOffset":2.4,"rankEligible":true,"rankMean":544.0911,"rankLo":485.0,"rankHi":608.0},{"stateId":"k-exaone-236b-a23b::disabled","configSlug":"k-exaone-236b-a23b-no-thinking","modelId":"k-exaone-236b-a23b","modelSlug":"k-exaone-236b-a23b","name":"K-EXAONE 236B-A23B (disabled)","modelName":"K-EXAONE 236B-A23B","organizationSlug":"lg-ai-research","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":135.99,"scoreLo":132.86,"scoreHi":139.13,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"k-exaone-236b-a23b::enabled","configSlug":"k-exaone-236b-a23b-thinking","modelId":"k-exaone-236b-a23b","modelSlug":"k-exaone-236b-a23b","name":"K-EXAONE 236B-A23B (thinking)","modelName":"K-EXAONE 236B-A23B","organizationSlug":"lg-ai-research","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":142.11,"scoreLo":139.13,"scoreHi":145.12,"reasoningOffset":2.13,"rankEligible":true,"rankMean":215.5597,"rankLo":189.975,"rankHi":245.0},{"stateId":"k2-horizon-0-9b::enabled","configSlug":"k2-horizon-0-9b-thinking","modelId":"k2-horizon-0-9b","modelSlug":"k2-horizon-0-9b","name":"K2 Horizon 0.9B (thinking)","modelName":"K2 Horizon 0.9B","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":117.37,"scoreLo":110.95,"scoreHi":122.88,"reasoningOffset":1.07,"rankEligible":true,"rankMean":534.1292,"rankLo":470.975,"rankHi":601.0},{"stateId":"k2-horizon-3-7b::enabled","configSlug":"k2-horizon-3-7b-thinking","modelId":"k2-horizon-3-7b","modelSlug":"k2-horizon-3-7b","name":"K2 Horizon 3.7B (thinking)","modelName":"K2 Horizon 3.7B","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":10,"score":141.71,"scoreLo":137.62,"scoreHi":145.62,"reasoningOffset":1.07,"rankEligible":true,"rankMean":220.7464,"rankLo":184.0,"rankHi":269.025},{"stateId":"k2-horizon-375b-a23b::enabled","configSlug":"k2-horizon-375b-a23b-thinking","modelId":"k2-horizon-375b-a23b","modelSlug":"k2-horizon-375b-a23b","name":"K2 Horizon 375B A23B (thinking)","modelName":"K2 Horizon 375B A23B","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":13,"score":153.75,"scoreLo":150.64,"scoreHi":157.39,"reasoningOffset":1.07,"rankEligible":true,"rankMean":95.3847,"rankLo":80.0,"rankHi":113.0},{"stateId":"k2-horizon-7b::enabled","configSlug":"k2-horizon-7b-thinking","modelId":"k2-horizon-7b","modelSlug":"k2-horizon-7b","name":"K2 Horizon 7B (thinking)","modelName":"K2 Horizon 7B","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":145.77,"scoreLo":142.82,"scoreHi":149.06,"reasoningOffset":1.07,"rankEligible":true,"rankMean":177.7975,"rankLo":152.0,"rankHi":205.0},{"stateId":"k2-horizon-mova-36b-a4b::enabled","configSlug":"k2-horizon-mova-36b-a4b-thinking","modelId":"k2-horizon-mova-36b-a4b","modelSlug":"k2-horizon-mova-36b-a4b","name":"K2 Horizon MoVA 36B A4B (thinking)","modelName":"K2 Horizon MoVA 36B A4B","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":150.38,"scoreLo":146.94,"scoreHi":154.13,"reasoningOffset":1.07,"rankEligible":true,"rankMean":127.7536,"rankLo":103.0,"rankHi":157.025},{"stateId":"k2-think::default","configSlug":"k2-think","modelId":"k2-think","modelSlug":"k2-think","name":"K2-Think","modelName":"K2-Think","organizationSlug":"llm360","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":5,"score":134.88,"scoreLo":130.94,"scoreHi":138.77,"reasoningOffset":2.4,"rankEligible":true,"rankMean":309.6661,"rankLo":260.0,"rankHi":358.0},{"stateId":"k2-think-v2::effort:high","configSlug":"k2-think-v2","modelId":"k2-think-v2","modelSlug":"k2-think-v2","name":"K2 Think V2 (high)","modelName":"K2 Think V2","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":true,"n":10,"score":134.26,"scoreLo":129.85,"scoreHi":138.25,"reasoningOffset":0.51,"rankEligible":true,"rankMean":318.0997,"rankLo":267.0,"rankHi":373.025},{"stateId":"k2-v2-instruct::effort:high","configSlug":"k2-v2-high","modelId":"k2-v2-instruct","modelSlug":"k2-v2-instruct","name":"K2-V2-Instruct (high)","modelName":"K2-V2-Instruct","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":8,"score":134.19,"scoreLo":130.89,"scoreHi":137.31,"reasoningOffset":1.7,"rankEligible":true,"rankMean":319.1367,"rankLo":279.0,"rankHi":359.0},{"stateId":"k2-v2-instruct::disabled","configSlug":"k2-v2-instruct-no-thinking","modelId":"k2-v2-instruct","modelSlug":"k2-v2-instruct","name":"K2-V2-Instruct (disabled)","modelName":"K2-V2-Instruct","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":128.5,"scoreLo":124.58,"scoreHi":132.26,"reasoningOffset":-3.99,"rankEligible":false},{"stateId":"k2-v2-instruct::effort:low","configSlug":"k2-v2-low","modelId":"k2-v2-instruct","modelSlug":"k2-v2-instruct","name":"K2-V2-Instruct (low)","modelName":"K2-V2-Instruct","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":8,"score":129.66,"scoreLo":126.21,"scoreHi":132.89,"reasoningOffset":-2.84,"rankEligible":true},{"stateId":"k2-v2-instruct::effort:medium","configSlug":"k2-v2-medium","modelId":"k2-v2-instruct","modelSlug":"k2-v2-instruct","name":"K2-V2-Instruct (medium)","modelName":"K2-V2-Instruct","organizationSlug":"llm360","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":8,"score":132.67,"scoreLo":129.29,"scoreHi":136.08,"reasoningOffset":0.18,"rankEligible":true},{"stateId":"kat-coder-pro::default","configSlug":"kat-coder-pro","modelId":"kat-coder-pro","modelSlug":"kat-coder-pro","name":"KAT-Coder-Pro V1","modelName":"KAT-Coder-Pro V1","organizationSlug":"kuaishou","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":140.63,"scoreLo":137.15,"scoreHi":144.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":232.9656,"rankLo":198.0,"rankHi":276.0},{"stateId":"kat-coder-pro-v2::default","configSlug":"kat-coder-pro-v2","modelId":"kat-coder-pro-v2","modelSlug":"kat-coder-pro-v2","name":"KAT-Coder-Pro V2","modelName":"KAT-Coder-Pro V2","organizationSlug":"kuaishou","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":149.83,"scoreLo":146.41,"scoreHi":153.54,"reasoningOffset":0.0,"rankEligible":true,"rankMean":133.6628,"rankLo":106.0,"rankHi":166.0},{"stateId":"kimi-k1-5::default","configSlug":"kimi-k1-5","modelId":"kimi-k1-5","modelSlug":"kimi-k1-5","name":"Kimi K1.5","modelName":"Kimi K1.5","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":127.42,"scoreLo":122.36,"scoreHi":132.32,"reasoningOffset":2.4,"rankEligible":true,"rankMean":406.6094,"rankLo":348.0,"rankHi":465.0},{"stateId":"kimi-k2-5::default","configSlug":"kimi-k2-5","modelId":"kimi-k2-5","modelSlug":"kimi-k2-5","name":"Kimi K2.5","modelName":"Kimi K2.5","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":101,"score":149.48,"scoreLo":147.24,"scoreHi":152.05,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"kimi-k2-5::effort:high","configSlug":"kimi-k2-5-high","modelId":"kimi-k2-5","modelSlug":"kimi-k2-5","name":"Kimi K2.5 (high)","modelName":"Kimi K2.5","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":5,"score":149.64,"scoreLo":146.7,"scoreHi":152.82,"reasoningOffset":0.15,"rankEligible":true},{"stateId":"kimi-k2-5::effort:minimal","configSlug":"kimi-k2-5-minimal","modelId":"kimi-k2-5","modelSlug":"kimi-k2-5","name":"Kimi K2.5 (minimal)","modelName":"Kimi K2.5","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":145.36,"scoreLo":140.6,"scoreHi":150.23,"reasoningOffset":-4.12,"rankEligible":false},{"stateId":"kimi-k2-5::disabled","configSlug":"kimi-k2-5-no-thinking","modelId":"kimi-k2-5","modelSlug":"kimi-k2-5","name":"Kimi K2.5 (disabled)","modelName":"Kimi K2.5","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":15,"score":146.37,"scoreLo":143.55,"scoreHi":149.46,"reasoningOffset":-3.12,"rankEligible":true},{"stateId":"kimi-k2-5::enabled","configSlug":"kimi-k2-5-thinking","modelId":"kimi-k2-5","modelSlug":"kimi-k2-5","name":"Kimi K2.5 (thinking)","modelName":"Kimi K2.5","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":90,"score":150.74,"scoreLo":148.46,"scoreHi":153.38,"reasoningOffset":1.26,"rankEligible":true,"rankMean":123.2983,"rankLo":115.0,"rankHi":133.0},{"stateId":"kimi-k2-6::default","configSlug":"kimi-k2-6","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":96,"score":153.3,"scoreLo":150.78,"scoreHi":156.23,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"kimi-k2-6::effort:64k","configSlug":"kimi-k2-6-64k","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (64k)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"64k","reasoningCondition":"effort:64k","isDefault":false,"n":7,"score":152.61,"scoreLo":150.04,"scoreHi":155.57,"reasoningOffset":-0.69,"rankEligible":true},{"stateId":"kimi-k2-6::effort:high","configSlug":"kimi-k2-6-high","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (high)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":8,"score":153.62,"scoreLo":150.81,"scoreHi":157.04,"reasoningOffset":0.32,"rankEligible":true},{"stateId":"kimi-k2-6::effort:low","configSlug":"kimi-k2-6-low","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (low)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":150.06,"scoreLo":145.68,"scoreHi":155.05,"reasoningOffset":-3.24,"rankEligible":false},{"stateId":"kimi-k2-6::effort:medium","configSlug":"kimi-k2-6-medium","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (medium)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":5,"score":151.68,"scoreLo":147.95,"scoreHi":155.69,"reasoningOffset":-1.63,"rankEligible":true},{"stateId":"kimi-k2-6::effort:minimal","configSlug":"kimi-k2-6-minimal","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (minimal)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":148.98,"scoreLo":144.4,"scoreHi":154.1,"reasoningOffset":-4.32,"rankEligible":false},{"stateId":"kimi-k2-6::disabled","configSlug":"kimi-k2-6-no-thinking","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (disabled)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":11,"score":149.12,"scoreLo":145.74,"scoreHi":152.97,"reasoningOffset":-4.18,"rankEligible":true},{"stateId":"kimi-k2-6::enabled","configSlug":"kimi-k2-6-thinking","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (thinking)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":74,"score":153.88,"scoreLo":151.33,"scoreHi":156.96,"reasoningOffset":0.58,"rankEligible":true,"rankMean":93.8803,"rankLo":87.0,"rankHi":101.0},{"stateId":"kimi-k2-6::effort:xhigh","configSlug":"kimi-k2-6-xhigh-genebench-tools","modelId":"kimi-k2-6","modelSlug":"kimi-k2-6","name":"Kimi K2.6 (xhigh)","modelName":"Kimi K2.6","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":154.32,"scoreLo":151.1,"scoreHi":158.21,"reasoningOffset":1.02,"rankEligible":false},{"stateId":"kimi-k2-7-code::default","configSlug":"kimi-k2-7-code","modelId":"kimi-k2-7-code","modelSlug":"kimi-k2-7-code","name":"Kimi K2.7 Code","modelName":"Kimi K2.7 Code","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":85,"score":154.34,"scoreLo":151.73,"scoreHi":157.52,"reasoningOffset":2.4,"rankEligible":true,"rankMean":89.7425,"rankLo":84.0,"rankHi":98.0},{"stateId":"kimi-k2-7-code::effort:high","configSlug":"kimi-k2-7-code-high-ramp-swe-bench-mini-swe-agent","modelId":"kimi-k2-7-code","modelSlug":"kimi-k2-7-code","name":"Kimi K2.7 Code (high)","modelName":"Kimi K2.7 Code","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":153.83,"scoreLo":150.22,"scoreHi":157.89,"reasoningOffset":1.89,"rankEligible":false},{"stateId":"kimi-k2-7-code::effort:minimal","configSlug":"kimi-k2-7-code-minimal","modelId":"kimi-k2-7-code","modelSlug":"kimi-k2-7-code","name":"Kimi K2.7 Code (minimal)","modelName":"Kimi K2.7 Code","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":147.63,"scoreLo":143.1,"scoreHi":152.79,"reasoningOffset":-4.31,"rankEligible":false},{"stateId":"kimi-k2-7-code::effort:xhigh","configSlug":"kimi-k2-7-code-xhigh-mini-swe-agent","modelId":"kimi-k2-7-code","modelSlug":"kimi-k2-7-code","name":"Kimi K2.7 Code (xhigh)","modelName":"Kimi K2.7 Code","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":152.98,"scoreLo":149.68,"scoreHi":156.86,"reasoningOffset":1.04,"rankEligible":false},{"stateId":"kimi-k2-base::default","configSlug":"kimi-k2-base","modelId":"kimi-k2-base","modelSlug":"kimi-k2-base","name":"Kimi K2 Base","modelName":"Kimi K2 Base","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":127.64,"scoreLo":124.47,"scoreHi":130.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":404.2667,"rankLo":371.0,"rankHi":434.0},{"stateId":"kimi-k2-instruct::default","configSlug":"kimi-k2-instruct","modelId":"kimi-k2-instruct","modelSlug":"kimi-k2-instruct","name":"Kimi K2 Instruct","modelName":"Kimi K2 Instruct","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":73,"score":139.72,"scoreLo":137.89,"scoreHi":141.65,"reasoningOffset":0.0,"rankEligible":true,"rankMean":243.6072,"rankLo":225.0,"rankHi":262.0},{"stateId":"kimi-k2-instruct-0905::default","configSlug":"kimi-k2-instruct-0905","modelId":"kimi-k2-instruct-0905","modelSlug":"kimi-k2-instruct-0905","name":"Kimi K2 Instruct 0905","modelName":"Kimi K2 Instruct 0905","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":42,"score":138.78,"scoreLo":136.64,"scoreHi":141.08,"reasoningOffset":0.0,"rankEligible":true,"rankMean":256.4411,"rankLo":232.0,"rankHi":281.0},{"stateId":"kimi-k2-thinking::default","configSlug":"kimi-k2-thinking","modelId":"kimi-k2-thinking","modelSlug":"kimi-k2-thinking","name":"Kimi K2 Thinking","modelName":"Kimi K2 Thinking","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":86,"score":145.56,"scoreLo":143.56,"scoreHi":147.82,"reasoningOffset":2.4,"rankEligible":true,"rankMean":180.0411,"rankLo":168.0,"rankHi":192.0},{"stateId":"kimi-k2-thinking::effort:minimal","configSlug":"kimi-k2-thinking-minimal","modelId":"kimi-k2-thinking","modelSlug":"kimi-k2-thinking","name":"Kimi K2 Thinking (minimal)","modelName":"Kimi K2 Thinking","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":139.07,"scoreLo":134.47,"scoreHi":143.97,"reasoningOffset":-4.09,"rankEligible":false},{"stateId":"kimi-k3::default","configSlug":"kimi-k3","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":85,"score":160.47,"scoreLo":157.22,"scoreHi":164.33,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"kimi-k3-1-pro::default","configSlug":"kimi-k3-1-pro","modelId":"kimi-k3-1-pro","modelSlug":"kimi-k3-1-pro","name":"Kimi K3.1 Pro","modelName":"Kimi K3.1 Pro","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":165.18,"scoreLo":161.32,"scoreHi":169.73,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"kimi-k3-1-pro::effort:high","configSlug":"kimi-k3-1-pro-high","modelId":"kimi-k3-1-pro","modelSlug":"kimi-k3-1-pro","name":"Kimi K3.1 Pro (high)","modelName":"Kimi K3.1 Pro","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":4,"score":164.55,"scoreLo":160.61,"scoreHi":169.29,"reasoningOffset":-0.63,"rankEligible":true},{"stateId":"kimi-k3-1-pro::effort:max","configSlug":"kimi-k3-1-pro-max","modelId":"kimi-k3-1-pro","modelSlug":"kimi-k3-1-pro","name":"Kimi K3.1 Pro (max)","modelName":"Kimi K3.1 Pro","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":38,"score":165.66,"scoreLo":161.76,"scoreHi":170.27,"reasoningOffset":0.48,"rankEligible":true,"rankMean":11.2256,"rankLo":9.0,"rankHi":14.0},{"stateId":"kimi-k3::effort:high","configSlug":"kimi-k3-high","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3 (high)","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":38,"score":159.96,"scoreLo":156.8,"scoreHi":163.77,"reasoningOffset":-0.51,"rankEligible":true},{"stateId":"kimi-k3::effort:low","configSlug":"kimi-k3-low","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3 (low)","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":30,"score":155.62,"scoreLo":152.82,"scoreHi":158.9,"reasoningOffset":-4.85,"rankEligible":true},{"stateId":"kimi-k3::effort:max","configSlug":"kimi-k3-max","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3 (max)","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":193,"score":160.49,"scoreLo":157.24,"scoreHi":164.27,"reasoningOffset":0.02,"rankEligible":true,"rankMean":43.9697,"rankLo":38.0,"rankHi":49.0},{"stateId":"kimi-k3::effort:medium","configSlug":"kimi-k3-medium","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3 (medium)","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":6,"score":158.29,"scoreLo":154.45,"scoreHi":162.57,"reasoningOffset":-2.18,"rankEligible":true},{"stateId":"kimi-k3::effort:minimal","configSlug":"kimi-k3-minimal","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3 (minimal)","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":156.26,"scoreLo":151.34,"scoreHi":161.83,"reasoningOffset":-4.21,"rankEligible":false},{"stateId":"kimi-k3::enabled","configSlug":"kimi-k3-reactbench-v1-mini-swe-agent","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3 (enabled)","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":3,"score":161.14,"scoreLo":157.36,"scoreHi":165.52,"reasoningOffset":0.67,"rankEligible":false},{"stateId":"kimi-k3::effort:xhigh","configSlug":"kimi-k3-xhigh","modelId":"kimi-k3","modelSlug":"kimi-k3","name":"Kimi K3 (xhigh)","modelName":"Kimi K3","organizationSlug":"moonshot-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":161.68,"scoreLo":157.97,"scoreHi":166.03,"reasoningOffset":1.21,"rankEligible":false},{"stateId":"kimi-linear-48b-a3b-base::default","configSlug":"kimi-linear-48b-a3b-base","modelId":"kimi-linear-48b-a3b-base","modelSlug":"kimi-linear-48b-a3b-base","name":"Kimi Linear 48B A3B Base","modelName":"Kimi Linear 48B A3B Base","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":117.75,"scoreLo":112.87,"scoreHi":122.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":529.8856,"rankLo":481.0,"rankHi":575.0},{"stateId":"kimi-linear-48b-a3b-instruct::default","configSlug":"kimi-linear-48b-a3b-instruct","modelId":"kimi-linear-48b-a3b-instruct","modelSlug":"kimi-linear-48b-a3b-instruct","name":"Kimi Linear 48B A3B Instruct","modelName":"Kimi Linear 48B A3B Instruct","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":130.69,"scoreLo":126.39,"scoreHi":134.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":365.6392,"rankLo":317.0,"rankHi":415.0},{"stateId":"kimi-vl-a3b-instruct::default","configSlug":"kimi-vl-a3b-instruct","modelId":"kimi-vl-a3b-instruct","modelSlug":"kimi-vl-a3b-instruct","name":"Kimi-VL-A3B-Instruct","modelName":"Kimi-VL-A3B-Instruct","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":19,"score":120.72,"scoreLo":116.47,"scoreHi":124.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":492.8236,"rankLo":450.0,"rankHi":531.0},{"stateId":"kimi-vl-a3b-thinking-2506::default","configSlug":"kimi-vl-a3b-thinking-2506-thinking","modelId":"kimi-vl-a3b-thinking-2506","modelSlug":"kimi-vl-a3b-thinking-2506","name":"Kimi-VL-A3B-Thinking-2506","modelName":"Kimi-VL-A3B-Thinking-2506","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":15,"score":126.03,"scoreLo":122.57,"scoreHi":129.48,"reasoningOffset":2.4,"rankEligible":true,"rankMean":423.2161,"rankLo":390.0,"rankHi":459.0},{"stateId":"kimi-vl-a3b-thinking::default","configSlug":"kimi-vl-a3b-thinking-thinking","modelId":"kimi-vl-a3b-thinking","modelSlug":"kimi-vl-a3b-thinking","name":"Kimi-VL-A3B-Thinking","modelName":"Kimi-VL-A3B-Thinking","organizationSlug":"moonshot-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":11,"score":120.61,"scoreLo":116.21,"scoreHi":124.65,"reasoningOffset":2.4,"rankEligible":true,"rankMean":494.3364,"rankLo":448.0,"rankHi":538.0},{"stateId":"laguna-m-1::default","configSlug":"laguna-m-1","modelId":"laguna-m-1","modelSlug":"laguna-m-1","name":"Laguna M.1","modelName":"Laguna M.1","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":139.13,"scoreLo":136.4,"scoreHi":141.83,"reasoningOffset":0.0,"rankEligible":true,"rankMean":251.7883,"rankLo":220.0,"rankHi":285.025},{"stateId":"laguna-m-1::effort:minimal","configSlug":"laguna-m-1-minimal","modelId":"laguna-m-1","modelSlug":"laguna-m-1","name":"Laguna M.1 (minimal)","modelName":"Laguna M.1","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":134.71,"scoreLo":129.54,"scoreHi":139.76,"reasoningOffset":-4.41,"rankEligible":false},{"stateId":"laguna-s-2-1::default","configSlug":"laguna-s-2-1-buseybench-svg-openrouter","modelId":"laguna-s-2-1","modelSlug":"laguna-s-2-1","name":"Laguna S 2.1","modelName":"Laguna S 2.1","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":false,"n":3,"score":150.9,"scoreLo":147.53,"scoreHi":154.55,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"laguna-s-2-1::enabled","configSlug":"laguna-s-2-1-thinking","modelId":"laguna-s-2-1","modelSlug":"laguna-s-2-1","name":"Laguna S 2.1 (thinking)","modelName":"Laguna S 2.1","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":8,"score":152.88,"scoreLo":149.08,"scoreHi":156.99,"reasoningOffset":1.98,"rankEligible":true,"rankMean":103.4567,"rankLo":80.0,"rankHi":133.0},{"stateId":"laguna-xs-2::default","configSlug":"laguna-xs-2","modelId":"laguna-xs-2","modelSlug":"laguna-xs-2","name":"Laguna XS.2","modelName":"Laguna XS.2","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":19,"score":136.05,"scoreLo":133.28,"scoreHi":138.8,"reasoningOffset":0.0,"rankEligible":true,"rankMean":293.6836,"rankLo":259.0,"rankHi":329.0},{"stateId":"laguna-xs-2-1::default","configSlug":"laguna-xs-2-1","modelId":"laguna-xs-2-1","modelSlug":"laguna-xs-2-1","name":"Laguna XS 2.1","modelName":"Laguna XS 2.1","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":137.28,"scoreLo":131.08,"scoreHi":142.89,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"laguna-xs-2-1::effort:minimal","configSlug":"laguna-xs-2-1-minimal","modelId":"laguna-xs-2-1","modelSlug":"laguna-xs-2-1","name":"Laguna XS 2.1 (minimal)","modelName":"Laguna XS 2.1","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":2,"score":133.34,"scoreLo":126.41,"scoreHi":139.43,"reasoningOffset":-3.94,"rankEligible":false},{"stateId":"laguna-xs-2-1::enabled","configSlug":"laguna-xs-2-1-thinking-context-arena-mrcr-v2","modelId":"laguna-xs-2-1","modelSlug":"laguna-xs-2-1","name":"Laguna XS 2.1 (thinking)","modelName":"Laguna XS 2.1","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":138.2,"scoreLo":131.45,"scoreHi":144.2,"reasoningOffset":0.92,"rankEligible":false},{"stateId":"laguna-xs-2::effort:minimal","configSlug":"laguna-xs-2-minimal","modelId":"laguna-xs-2","modelSlug":"laguna-xs-2","name":"Laguna XS.2 (minimal)","modelName":"Laguna XS.2","organizationSlug":"poolside","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":131.44,"scoreLo":126.08,"scoreHi":136.84,"reasoningOffset":-4.62,"rankEligible":false},{"stateId":"lfm2-1-2b::default","configSlug":"lfm2-1-2b","modelId":"lfm2-1-2b","modelSlug":"lfm2-1-2b","name":"LFM2-1.2B","modelName":"LFM2-1.2B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":92.79,"scoreLo":83.6,"scoreHi":100.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":772.2739,"rankLo":729.0,"rankHi":807.0},{"stateId":"lfm2-2-6b::disabled","configSlug":"lfm2-2-6b-no-thinking","modelId":"lfm2-2-6b","modelSlug":"lfm2-2-6b","name":"LFM2-2.6B (disabled)","modelName":"LFM2-2.6B","organizationSlug":"liquid-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":97.55,"scoreLo":88.1,"scoreHi":105.6,"reasoningOffset":-3.99,"rankEligible":true,"rankMean":737.4272,"rankLo":684.0,"rankHi":789.0},{"stateId":"lfm2-24b-a2b::default","configSlug":"lfm2-24b-a2b","modelId":"lfm2-24b-a2b","modelSlug":"lfm2-24b-a2b","name":"LFM2-24B-A2B","modelName":"LFM2-24B-A2B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":113.94,"scoreLo":97.03,"scoreHi":128.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":571.5697,"rankLo":399.0,"rankHi":736.025},{"stateId":"lfm2-350m::default","configSlug":"lfm2-350m","modelId":"lfm2-350m","modelSlug":"lfm2-350m","name":"LFM2-350M","modelName":"LFM2-350M","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":81.57,"scoreLo":70.68,"scoreHi":90.54,"reasoningOffset":0.0,"rankEligible":true,"rankMean":827.1678,"rankLo":802.0,"rankHi":851.0},{"stateId":"lfm2-5-1-2b-instruct::default","configSlug":"lfm2-5-1-2b-instruct","modelId":"lfm2-5-1-2b-instruct","modelSlug":"lfm2-5-1-2b-instruct","name":"LFM2.5-1.2B-Instruct","modelName":"LFM2.5-1.2B-Instruct","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":108.51,"scoreLo":93.57,"scoreHi":120.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":633.6219,"rankLo":498.0,"rankHi":764.0},{"stateId":"lfm2-5-1-2b-thinking::default","configSlug":"lfm2-5-1-2b-thinking","modelId":"lfm2-5-1-2b-thinking","modelSlug":"lfm2-5-1-2b-thinking","name":"LFM2.5-1.2B-Thinking","modelName":"LFM2.5-1.2B-Thinking","organizationSlug":"liquid-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":5,"score":113.36,"scoreLo":96.26,"scoreHi":128.5,"reasoningOffset":2.4,"rankEligible":true,"rankMean":577.9411,"rankLo":395.0,"rankHi":744.025},{"stateId":"lfm2-5-2-6b::default","configSlug":"lfm2-5-2-6b","modelId":"lfm2-5-2-6b","modelSlug":"lfm2-5-2-6b","name":"LFM2.5-2.6B","modelName":"LFM2.5-2.6B","organizationSlug":"liquid-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":123.41,"scoreLo":118.98,"scoreHi":127.41,"reasoningOffset":2.4,"rankEligible":true,"rankMean":456.4819,"rankLo":412.0,"rankHi":510.0},{"stateId":"lfm2-5-230m::default","configSlug":"lfm2-5-230m","modelId":"lfm2-5-230m","modelSlug":"lfm2-5-230m","name":"LFM2.5-230M","modelName":"LFM2.5-230M","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":92.73,"scoreLo":82.23,"scoreHi":101.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":771.7181,"rankLo":720.0,"rankHi":813.0},{"stateId":"lfm2-5-24b-a2b::default","configSlug":"lfm2-5-24b-a2b","modelId":"lfm2-5-24b-a2b","modelSlug":"lfm2-5-24b-a2b","name":"LFM2.5-24B-A2B","modelName":"LFM2.5-24B-A2B","organizationSlug":"liquid-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":139.26,"scoreLo":135.68,"scoreHi":142.89,"reasoningOffset":2.4,"rankEligible":true,"rankMean":250.4314,"rankLo":211.0,"rankHi":296.0},{"stateId":"lfm2-5-350m::default","configSlug":"lfm2-5-350m","modelId":"lfm2-5-350m","modelSlug":"lfm2-5-350m","name":"LFM2.5-350M","modelName":"LFM2.5-350M","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":95.09,"scoreLo":85.1,"scoreHi":103.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":755.7056,"rankLo":703.0,"rankHi":803.0},{"stateId":"lfm2-5-8b-a1b::default","configSlug":"lfm2-5-8b-a1b","modelId":"lfm2-5-8b-a1b","modelSlug":"lfm2-5-8b-a1b","name":"LFM2.5-8B-A1B","modelName":"LFM2.5-8B-A1B","organizationSlug":"liquid-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":123.64,"scoreLo":113.26,"scoreHi":132.32,"reasoningOffset":2.4,"rankEligible":true,"rankMean":454.2406,"rankLo":345.975,"rankHi":578.0},{"stateId":"lfm2-5-vl-1-6b::default","configSlug":"lfm2-5-vl-1-6b","modelId":"lfm2-5-vl-1-6b","modelSlug":"lfm2-5-vl-1-6b","name":"LFM2.5-VL-1.6B","modelName":"LFM2.5-VL-1.6B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":103.41,"scoreLo":91.6,"scoreHi":112.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":687.4628,"rankLo":598.975,"rankHi":771.0},{"stateId":"lfm2-5-vl-3b::default","configSlug":"lfm2-5-vl-3b","modelId":"lfm2-5-vl-3b","modelSlug":"lfm2-5-vl-3b","name":"LFM2.5-VL-3B","modelName":"LFM2.5-VL-3B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":48,"score":114.12,"scoreLo":109.51,"scoreHi":118.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":572.5953,"rankLo":542.0,"rankHi":606.0},{"stateId":"lfm2-5-vl-450m::default","configSlug":"lfm2-5-vl-450m","modelId":"lfm2-5-vl-450m","modelSlug":"lfm2-5-vl-450m","name":"LFM2.5-VL-450M","modelName":"LFM2.5-VL-450M","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":89.58,"scoreLo":80.5,"scoreHi":97.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":791.76,"rankLo":763.0,"rankHi":818.0},{"stateId":"lfm2-8b-a1b::default","configSlug":"lfm2-8b-a1b","modelId":"lfm2-8b-a1b","modelSlug":"lfm2-8b-a1b","name":"LFM2-8B-A1B","modelName":"LFM2-8B-A1B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":111.56,"scoreLo":103.8,"scoreHi":118.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":603.6672,"rankLo":531.0,"rankHi":678.0},{"stateId":"lfm2-vl-1-6b::default","configSlug":"lfm2-vl-1-6b","modelId":"lfm2-vl-1-6b","modelSlug":"lfm2-vl-1-6b","name":"LFM2-VL-1.6B","modelName":"LFM2-VL-1.6B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":98.01,"scoreLo":90.59,"scoreHi":104.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":734.3961,"rankLo":699.0,"rankHi":772.0},{"stateId":"lfm2-vl-3b::default","configSlug":"lfm2-vl-3b","modelId":"lfm2-vl-3b","modelSlug":"lfm2-vl-3b","name":"LFM2-VL-3B","modelName":"LFM2-VL-3B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":110.91,"scoreLo":104.95,"scoreHi":116.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":612.0931,"rankLo":553.0,"rankHi":665.0},{"stateId":"lfm2-vl-450m::default","configSlug":"lfm2-vl-450m","modelId":"lfm2-vl-450m","modelSlug":"lfm2-vl-450m","name":"LFM2-VL-450M","modelName":"LFM2-VL-450M","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":82.46,"scoreLo":72.3,"scoreHi":90.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":824.1442,"rankLo":804.0,"rankHi":844.0},{"stateId":"lfm3-1-7b::default","configSlug":"lfm3-1-7b","modelId":"lfm3-1-7b","modelSlug":"lfm3-1-7b","name":"LFM3-1.7B","modelName":"LFM3-1.7B","organizationSlug":"liquid-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":136.84,"scoreLo":133.45,"scoreHi":140.11,"reasoningOffset":0.0,"rankEligible":true,"rankMean":283.0044,"rankLo":243.0,"rankHi":326.0},{"stateId":"lfm3-omni-2b::default","configSlug":"lfm3-omni-2b","modelId":"lfm3-omni-2b","modelSlug":"lfm3-omni-2b","name":"LFM3-Omni-2B","modelName":"LFM3-Omni-2B","organizationSlug":"liquid-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":110.19,"scoreLo":103.47,"scoreHi":116.0,"reasoningOffset":0.0,"rankEligible":true,"rankMean":620.3178,"rankLo":561.0,"rankHi":676.0},{"stateId":"lfm3-vl-2b::disabled","configSlug":"lfm3-vl-2b-no-thinking","modelId":"lfm3-vl-2b","modelSlug":"lfm3-vl-2b","name":"LFM3-VL-2B (disabled)","modelName":"LFM3-VL-2B","organizationSlug":"liquid-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":49,"score":123.23,"scoreLo":119.75,"scoreHi":126.46,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"lfm3-vl-2b::enabled","configSlug":"lfm3-vl-2b-thinking","modelId":"lfm3-vl-2b","modelSlug":"lfm3-vl-2b","name":"LFM3-VL-2B (thinking)","modelName":"LFM3-VL-2B","organizationSlug":"liquid-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":49,"score":129.98,"scoreLo":127.37,"scoreHi":132.51,"reasoningOffset":2.77,"rankEligible":true,"rankMean":375.1811,"rankLo":347.0,"rankHi":402.0},{"stateId":"ling-1t::default","configSlug":"ling-1t","modelId":"ling-1t","modelSlug":"ling-1t","name":"Ling-1T","modelName":"Ling-1T","organizationSlug":"inclusion-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":137.54,"scoreLo":133.45,"scoreHi":141.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":273.5458,"rankLo":223.0,"rankHi":329.0},{"stateId":"ling-2-6-1t::default","configSlug":"ling-2-6-1t","modelId":"ling-2-6-1t","modelSlug":"ling-2-6-1t","name":"Ling 2.6 1T","modelName":"Ling 2.6 1T","organizationSlug":"inclusion-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":140.39,"scoreLo":135.83,"scoreHi":144.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":236.4431,"rankLo":194.0,"rankHi":292.0},{"stateId":"ling-2-6-flash::default","configSlug":"ling-2-6-flash","modelId":"ling-2-6-flash","modelSlug":"ling-2-6-flash","name":"Ling 2.6 Flash","modelName":"Ling 2.6 Flash","organizationSlug":"inclusion-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":134.41,"scoreLo":129.45,"scoreHi":138.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":315.9125,"rankLo":261.0,"rankHi":382.0},{"stateId":"ling-3-0-flash::default","configSlug":"ling-3-0-flash","modelId":"ling-3-0-flash","modelSlug":"ling-3-0-flash","name":"Ling 3.0 Flash","modelName":"Ling 3.0 Flash","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":146.01,"scoreLo":143.52,"scoreHi":148.65,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"ling-3-0-flash-fin::default","configSlug":"ling-3-0-flash-fin-samaya-frontierfinance-v3-1-fa-v2","modelId":"ling-3-0-flash-fin","modelSlug":"ling-3-0-flash-fin","name":"Ling 3.0 Flash Fin","modelName":"Ling 3.0 Flash Fin","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":1,"score":149.54,"scoreLo":145.35,"scoreHi":154.18,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"ling-3-0-flash-fin::enabled","configSlug":"ling-3-0-flash-fin-thinking","modelId":"ling-3-0-flash-fin","modelSlug":"ling-3-0-flash-fin","name":"Ling 3.0 Flash Fin (thinking)","modelName":"Ling 3.0 Flash Fin","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":150.21,"scoreLo":146.97,"scoreHi":153.86,"reasoningOffset":0.68,"rankEligible":true,"rankMean":129.5356,"rankLo":106.0,"rankHi":157.0},{"stateId":"ling-3-0-flash::enabled","configSlug":"ling-3-0-flash-thinking","modelId":"ling-3-0-flash","modelSlug":"ling-3-0-flash","name":"Ling 3.0 Flash (thinking)","modelName":"Ling 3.0 Flash","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":147.79,"scoreLo":144.9,"scoreHi":150.86,"reasoningOffset":1.78,"rankEligible":true,"rankMean":155.9606,"rankLo":132.0,"rankHi":179.025},{"stateId":"ling-3-0-flash-vl::default","configSlug":"ling-3-0-flash-vl","modelId":"ling-3-0-flash-vl","modelSlug":"ling-3-0-flash-vl","name":"Ling 3.0 Flash VL","modelName":"Ling 3.0 Flash VL","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":2,"score":150.06,"scoreLo":145.82,"scoreHi":154.55,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"ling-3-0-flash-vl::enabled","configSlug":"ling-3-0-flash-vl-thinking","modelId":"ling-3-0-flash-vl","modelSlug":"ling-3-0-flash-vl","name":"Ling 3.0 Flash VL (thinking)","modelName":"Ling 3.0 Flash VL","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":151.49,"scoreLo":148.34,"scoreHi":155.14,"reasoningOffset":1.43,"rankEligible":true,"rankMean":116.1964,"rankLo":95.0,"rankHi":140.0},{"stateId":"ling-3-0-tiny::enabled","configSlug":"ling-3-0-tiny-thinking","modelId":"ling-3-0-tiny","modelSlug":"ling-3-0-tiny","name":"Ling 3.0 Tiny (thinking)","modelName":"Ling 3.0 Tiny","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":10,"score":138.41,"scoreLo":134.67,"scoreHi":141.95,"reasoningOffset":1.07,"rankEligible":true,"rankMean":261.7239,"rankLo":219.0,"rankHi":309.0},{"stateId":"ling-3-1-flash::default","configSlug":"ling-3-1-flash","modelId":"ling-3-1-flash","modelSlug":"ling-3-1-flash","name":"Ling 3.1 Flash","modelName":"Ling 3.1 Flash","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":1,"score":160.32,"scoreLo":155.85,"scoreHi":165.55,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"ling-3-1-flash-1019::enabled","configSlug":"ling-3-1-flash-1019-thinking","modelId":"ling-3-1-flash-1019","modelSlug":"ling-3-1-flash-1019","name":"Ling 3.1 Flash 1019 (thinking)","modelName":"Ling 3.1 Flash 1019","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":10,"score":161.05,"scoreLo":157.5,"scoreHi":165.25,"reasoningOffset":1.07,"rankEligible":true,"rankMean":37.7272,"rankLo":25.0,"rankHi":52.0},{"stateId":"ling-3-1-flash::enabled","configSlug":"ling-3-1-flash-reasoning","modelId":"ling-3-1-flash","modelSlug":"ling-3-1-flash","name":"Ling 3.1 Flash (enabled)","modelName":"Ling 3.1 Flash","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":8,"score":160.85,"scoreLo":156.86,"scoreHi":165.42,"reasoningOffset":0.53,"rankEligible":true,"rankMean":39.9522,"rankLo":22.0,"rankHi":63.0},{"stateId":"ling-flash-2-0::default","configSlug":"ling-flash-2-0","modelId":"ling-flash-2-0","modelSlug":"ling-flash-2-0","name":"Ling Flash 2.0","modelName":"Ling Flash 2.0","organizationSlug":"inclusion-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":132.12,"scoreLo":127.62,"scoreHi":136.25,"reasoningOffset":0.0,"rankEligible":true,"rankMean":346.7458,"rankLo":294.0,"rankHi":401.0},{"stateId":"ling-mini-2-0::default","configSlug":"ling-mini-2-0","modelId":"ling-mini-2-0","modelSlug":"ling-mini-2-0","name":"Ling Mini 2.0","modelName":"Ling Mini 2.0","organizationSlug":"inclusion-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":123.36,"scoreLo":117.47,"scoreHi":128.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":457.6194,"rankLo":397.0,"rankHi":526.0},{"stateId":"llama-13b::default","configSlug":"llama-13b","modelId":"llama-13b","modelSlug":"llama-13b","name":"LLaMA-13B","modelName":"LLaMA-13B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":80.75,"scoreLo":71.05,"scoreHi":88.88,"reasoningOffset":0.0,"rankEligible":true,"rankMean":830.6286,"rankLo":814.0,"rankHi":846.0},{"stateId":"llama-2-13b-chat-hf::default","configSlug":"llama-2-13b-chat-hf","modelId":"llama-2-13b-chat-hf","modelSlug":"llama-2-13b-chat-hf","name":"Llama 2 13B Chat","modelName":"Llama 2 13B Chat","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":89.22,"scoreLo":81.06,"scoreHi":95.78,"reasoningOffset":0.0,"rankEligible":true,"rankMean":794.47,"rankLo":776.0,"rankHi":810.0},{"stateId":"llama-2-13b-hf::default","configSlug":"llama-2-13b-hf","modelId":"llama-2-13b-hf","modelSlug":"llama-2-13b-hf","name":"Llama 2 13B","modelName":"Llama 2 13B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":37,"score":88.29,"scoreLo":80.26,"scoreHi":95.02,"reasoningOffset":0.0,"rankEligible":true,"rankMean":799.3797,"rankLo":784.0,"rankHi":814.0},{"stateId":"llama-2-70b-chat-hf::default","configSlug":"llama-2-70b-chat-hf","modelId":"llama-2-70b-chat-hf","modelSlug":"llama-2-70b-chat-hf","name":"Llama 2 70B Chat","modelName":"Llama 2 70B Chat","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":96.07,"scoreLo":89.03,"scoreHi":101.94,"reasoningOffset":0.0,"rankEligible":true,"rankMean":750.1319,"rankLo":727.0,"rankHi":771.0},{"stateId":"llama-2-70b-hf::default","configSlug":"llama-2-70b-hf","modelId":"llama-2-70b-hf","modelSlug":"llama-2-70b-hf","name":"Llama 2 70B","modelName":"Llama 2 70B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":34,"score":104.25,"scoreLo":98.59,"scoreHi":109.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":683.3,"rankLo":659.0,"rankHi":706.0},{"stateId":"llama-2-7b-chat-hf::default","configSlug":"llama-2-7b-chat-hf","modelId":"llama-2-7b-chat-hf","modelSlug":"llama-2-7b-chat-hf","name":"Llama 2 7B Chat","modelName":"Llama 2 7B Chat","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":81.83,"scoreLo":72.77,"scoreHi":89.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":826.7597,"rankLo":814.0,"rankHi":839.0},{"stateId":"llama-2-7b-hf::default","configSlug":"llama-2-7b-hf","modelId":"llama-2-7b-hf","modelSlug":"llama-2-7b-hf","name":"Llama 2 7B","modelName":"Llama 2 7B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":37,"score":79.62,"scoreLo":70.27,"scoreHi":87.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":834.8322,"rankLo":823.0,"rankHi":845.025},{"stateId":"llama-3-1-405b::default","configSlug":"llama-3-1-405b","modelId":"llama-3-1-405b","modelSlug":"llama-3-1-405b","name":"Llama 3.1 405B","modelName":"Llama 3.1 405B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":120.33,"scoreLo":116.35,"scoreHi":123.88,"reasoningOffset":0.0,"rankEligible":true,"rankMean":498.2264,"rankLo":459.0,"rankHi":532.0},{"stateId":"llama-3-1-405b-instruct::default","configSlug":"llama-3-1-405b-instruct","modelId":"llama-3-1-405b-instruct","modelSlug":"llama-3-1-405b-instruct","name":"Llama 3.1 405B Instruct","modelName":"Llama 3.1 405B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":85,"score":125.77,"scoreLo":123.35,"scoreHi":127.93,"reasoningOffset":0.0,"rankEligible":true,"rankMean":425.8453,"rankLo":408.0,"rankHi":445.0},{"stateId":"llama-3-1-70b::default","configSlug":"llama-3-1-70b","modelId":"llama-3-1-70b","modelSlug":"llama-3-1-70b","name":"Llama 3.1 70B","modelName":"Llama 3.1 70B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":116.94,"scoreLo":113.02,"scoreHi":120.39,"reasoningOffset":0.0,"rankEligible":true,"rankMean":539.3419,"rankLo":510.0,"rankHi":569.0},{"stateId":"llama-3-1-70b-instruct::default","configSlug":"llama-3-1-70b-instruct","modelId":"llama-3-1-70b-instruct","modelSlug":"llama-3-1-70b-instruct","name":"Llama 3.1 70B Instruct","modelName":"Llama 3.1 70B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":70,"score":121.8,"scoreLo":118.81,"scoreHi":124.36,"reasoningOffset":0.0,"rankEligible":true,"rankMean":478.1181,"rankLo":455.0,"rankHi":502.0},{"stateId":"llama-3-1-8b::default","configSlug":"llama-3-1-8b","modelId":"llama-3-1-8b","modelSlug":"llama-3-1-8b","name":"Llama 3.1 8B","modelName":"Llama 3.1 8B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":99.81,"scoreLo":93.42,"scoreHi":105.42,"reasoningOffset":0.0,"rankEligible":true,"rankMean":719.9322,"rankLo":696.0,"rankHi":745.0},{"stateId":"llama-3-1-8b-instruct::default","configSlug":"llama-3-1-8b-instruct","modelId":"llama-3-1-8b-instruct","modelSlug":"llama-3-1-8b-instruct","name":"Llama 3.1 8B Instruct","modelName":"Llama 3.1 8B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":89,"score":107.61,"scoreLo":102.86,"scoreHi":111.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":650.5833,"rankLo":631.0,"rankHi":669.0},{"stateId":"llama-3-1-nemotron-51b-instruct::default","configSlug":"llama-3-1-nemotron-51b-instruct","modelId":"llama-3-1-nemotron-51b-instruct","modelSlug":"llama-3-1-nemotron-51b-instruct","name":"Llama 3.1 Nemotron 51B Instruct","modelName":"Llama 3.1 Nemotron 51B Instruct","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":117.19,"scoreLo":109.3,"scoreHi":124.82,"reasoningOffset":0.0,"rankEligible":true,"rankMean":535.7703,"rankLo":443.0,"rankHi":621.0},{"stateId":"llama-3-1-nemotron-70b-instruct-hf::default","configSlug":"llama-3-1-nemotron-70b-instruct-hf","modelId":"llama-3-1-nemotron-70b-instruct-hf","modelSlug":"llama-3-1-nemotron-70b-instruct-hf","name":"Llama 3.1 Nemotron 70B Instruct HF","modelName":"Llama 3.1 Nemotron 70B Instruct HF","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":121.6,"scoreLo":117.99,"scoreHi":124.99,"reasoningOffset":0.0,"rankEligible":true,"rankMean":481.0103,"rankLo":445.0,"rankHi":515.0},{"stateId":"llama-3-1-nemotron-nano-4b-v1-1::disabled","configSlug":"llama-3-1-nemotron-nano-4b-v1-1-no-thinking","modelId":"llama-3-1-nemotron-nano-4b-v1-1","modelSlug":"llama-3-1-nemotron-nano-4b-v1-1","name":"Llama 3.1 Nemotron Nano 4B V1.1 (disabled)","modelName":"Llama 3.1 Nemotron Nano 4B V1.1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":3,"score":112.58,"scoreLo":105.23,"scoreHi":119.09,"reasoningOffset":-3.99,"rankEligible":false},{"stateId":"llama-3-1-nemotron-nano-4b-v1-1::enabled","configSlug":"llama-3-1-nemotron-nano-4b-v1-1-thinking","modelId":"llama-3-1-nemotron-nano-4b-v1-1","modelSlug":"llama-3-1-nemotron-nano-4b-v1-1","name":"Llama 3.1 Nemotron Nano 4B V1.1 (thinking)","modelName":"Llama 3.1 Nemotron Nano 4B V1.1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":118.59,"scoreLo":112.43,"scoreHi":124.2,"reasoningOffset":2.01,"rankEligible":true,"rankMean":519.2708,"rankLo":452.0,"rankHi":582.0},{"stateId":"llama-3-1-nemotron-nano-8b-v1::disabled","configSlug":"llama-3-1-nemotron-nano-8b-v1-no-thinking","modelId":"llama-3-1-nemotron-nano-8b-v1","modelSlug":"llama-3-1-nemotron-nano-8b-v1","name":"Llama 3.1 Nemotron Nano 8B V1 (disabled)","modelName":"Llama 3.1 Nemotron Nano 8B V1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":4,"score":118.65,"scoreLo":111.54,"scoreHi":124.79,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"llama-3-1-nemotron-nano-8b-v1::enabled","configSlug":"llama-3-1-nemotron-nano-8b-v1-thinking","modelId":"llama-3-1-nemotron-nano-8b-v1","modelSlug":"llama-3-1-nemotron-nano-8b-v1","name":"Llama 3.1 Nemotron Nano 8B V1 (thinking)","modelName":"Llama 3.1 Nemotron Nano 8B V1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":6,"score":124.73,"scoreLo":119.31,"scoreHi":129.77,"reasoningOffset":2.09,"rankEligible":true,"rankMean":440.0372,"rankLo":384.0,"rankHi":508.0},{"stateId":"llama-3-1-nemotron-ultra-253b-v1::default","configSlug":"llama-3-1-nemotron-ultra-253b-v1","modelId":"llama-3-1-nemotron-ultra-253b-v1","modelSlug":"llama-3-1-nemotron-ultra-253b-v1","name":"Llama 3.1 Nemotron Ultra 253B V1","modelName":"Llama 3.1 Nemotron Ultra 253B V1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":5,"score":131.08,"scoreLo":127.18,"scoreHi":134.74,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"llama-3-1-nemotron-ultra-253b-v1::disabled","configSlug":"llama-3-1-nemotron-ultra-253b-v1-no-thinking","modelId":"llama-3-1-nemotron-ultra-253b-v1","modelSlug":"llama-3-1-nemotron-ultra-253b-v1","name":"Llama 3.1 Nemotron Ultra 253B V1 (disabled)","modelName":"Llama 3.1 Nemotron Ultra 253B V1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":3,"score":128.24,"scoreLo":124.41,"scoreHi":131.65,"reasoningOffset":-2.84,"rankEligible":false},{"stateId":"llama-3-1-nemotron-ultra-253b-v1::enabled","configSlug":"llama-3-1-nemotron-ultra-253b-v1-thinking","modelId":"llama-3-1-nemotron-ultra-253b-v1","modelSlug":"llama-3-1-nemotron-ultra-253b-v1","name":"Llama 3.1 Nemotron Ultra 253B V1 (thinking)","modelName":"Llama 3.1 Nemotron Ultra 253B V1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":133.55,"scoreLo":130.32,"scoreHi":136.51,"reasoningOffset":2.47,"rankEligible":true,"rankMean":327.9536,"rankLo":290.0,"rankHi":364.0},{"stateId":"llama-3-1-tulu-3-70b-dpo::default","configSlug":"llama-3-1-tulu-3-70b-dpo","modelId":"llama-3-1-tulu-3-70b-dpo","modelSlug":"llama-3-1-tulu-3-70b-dpo","name":"Llama 3.1 T\u00fclu 3 70B DPO","modelName":"Llama 3.1 T\u00fclu 3 70B DPO","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":120.16,"scoreLo":115.04,"scoreHi":124.78,"reasoningOffset":0.0,"rankEligible":true,"rankMean":499.7558,"rankLo":447.0,"rankHi":551.0},{"stateId":"llama-3-1-tulu-3-8b::default","configSlug":"llama-3-1-tulu-3-8b","modelId":"llama-3-1-tulu-3-8b","modelSlug":"llama-3-1-tulu-3-8b","name":"Llama-3.1-T\u00fclu-3-8B","modelName":"Llama-3.1-T\u00fclu-3-8B","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":100.87,"scoreLo":93.23,"scoreHi":107.49,"reasoningOffset":0.0,"rankEligible":true,"rankMean":711.3858,"rankLo":669.0,"rankHi":756.0},{"stateId":"llama-3-2-11b-vision-instruct::default","configSlug":"llama-3-2-11b-vision-instruct","modelId":"llama-3-2-11b-vision-instruct","modelSlug":"llama-3-2-11b-vision-instruct","name":"Llama 3.2 11B Vision Instruct","modelName":"Llama 3.2 11B Vision Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":38,"score":106.79,"scoreLo":101.49,"scoreHi":111.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":658.9675,"rankLo":629.0,"rankHi":686.0},{"stateId":"llama-3-2-1b::default","configSlug":"llama-3-2-1b","modelId":"llama-3-2-1b","modelSlug":"llama-3-2-1b","name":"Llama 3.2 1B","modelName":"Llama 3.2 1B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":62.61,"scoreLo":47.8,"scoreHi":75.63,"reasoningOffset":0.0,"rankEligible":true,"rankMean":869.2053,"rankLo":858.0,"rankHi":879.0},{"stateId":"llama-3-2-1b-instruct::default","configSlug":"llama-3-2-1b-instruct","modelId":"llama-3-2-1b-instruct","modelSlug":"llama-3-2-1b-instruct","name":"Llama 3.2 1B Instruct","modelName":"Llama 3.2 1B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":42,"score":84.41,"scoreLo":75.65,"scoreHi":91.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":816.5803,"rankLo":802.0,"rankHi":831.0},{"stateId":"llama-3-2-3b::default","configSlug":"llama-3-2-3b","modelId":"llama-3-2-3b","modelSlug":"llama-3-2-3b","name":"Llama 3.2 3B","modelName":"Llama 3.2 3B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":92.81,"scoreLo":83.93,"scoreHi":100.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":772.3403,"rankLo":735.0,"rankHi":805.0},{"stateId":"llama-3-2-3b-instruct::default","configSlug":"llama-3-2-3b-instruct","modelId":"llama-3-2-3b-instruct","modelSlug":"llama-3-2-3b-instruct","name":"Llama 3.2 3B Instruct","modelName":"Llama 3.2 3B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":43,"score":102.52,"scoreLo":96.69,"scoreHi":107.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":698.3542,"rankLo":678.0,"rankHi":718.0},{"stateId":"llama-3-2-90b-vision-instruct::default","configSlug":"llama-3-2-90b-vision-instruct","modelId":"llama-3-2-90b-vision-instruct","modelSlug":"llama-3-2-90b-vision-instruct","name":"Llama 3.2 90B Vision Instruct","modelName":"Llama 3.2 90B Vision Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":121.93,"scoreLo":118.38,"scoreHi":125.11,"reasoningOffset":0.0,"rankEligible":true,"rankMean":476.1786,"rankLo":441.975,"rankHi":511.0},{"stateId":"llama-3-3-70b-instruct::default","configSlug":"llama-3-3-70b-instruct","modelId":"llama-3-3-70b-instruct","modelSlug":"llama-3-3-70b-instruct","name":"Llama 3.3 70B Instruct","modelName":"Llama 3.3 70B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":67,"score":124.45,"scoreLo":121.73,"scoreHi":126.97,"reasoningOffset":0.0,"rankEligible":true,"rankMean":442.3767,"rankLo":420.0,"rankHi":465.0},{"stateId":"llama-3-3-nemotron-super-49b-v1-5::default","configSlug":"llama-3-3-nemotron-super-49b-v1-5","modelId":"llama-3-3-nemotron-super-49b-v1-5","modelSlug":"llama-3-3-nemotron-super-49b-v1-5","name":"Llama 3.3 Nemotron Super 49B V1.5","modelName":"Llama 3.3 Nemotron Super 49B V1.5","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":1,"score":132.23,"scoreLo":128.08,"scoreHi":136.15,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"llama-3-3-nemotron-super-49b-v1-5::disabled","configSlug":"llama-3-3-nemotron-super-49b-v1-5-no-thinking","modelId":"llama-3-3-nemotron-super-49b-v1-5","modelSlug":"llama-3-3-nemotron-super-49b-v1-5","name":"Llama 3.3 Nemotron Super 49B V1.5 (disabled)","modelName":"Llama 3.3 Nemotron Super 49B V1.5","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":127.98,"scoreLo":123.91,"scoreHi":131.67,"reasoningOffset":-4.25,"rankEligible":true},{"stateId":"llama-3-3-nemotron-super-49b-v1-5::enabled","configSlug":"llama-3-3-nemotron-super-49b-v1-5-thinking","modelId":"llama-3-3-nemotron-super-49b-v1-5","modelSlug":"llama-3-3-nemotron-super-49b-v1-5","name":"Llama 3.3 Nemotron Super 49B V1.5 (thinking)","modelName":"Llama 3.3 Nemotron Super 49B V1.5","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":13,"score":135.15,"scoreLo":131.88,"scoreHi":138.32,"reasoningOffset":2.92,"rankEligible":true,"rankMean":306.0681,"rankLo":266.0,"rankHi":347.0},{"stateId":"llama-3-3-nemotron-super-49b-v1::disabled","configSlug":"llama-3-3-nemotron-super-49b-v1-no-thinking","modelId":"llama-3-3-nemotron-super-49b-v1","modelSlug":"llama-3-3-nemotron-super-49b-v1","name":"Llama 3.3 Nemotron Super 49B V1 (disabled)","modelName":"Llama 3.3 Nemotron Super 49B V1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":13,"score":126.29,"scoreLo":122.8,"scoreHi":129.47,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"llama-3-3-nemotron-super-49b-v1::enabled","configSlug":"llama-3-3-nemotron-super-49b-v1-thinking","modelId":"llama-3-3-nemotron-super-49b-v1","modelSlug":"llama-3-3-nemotron-super-49b-v1","name":"Llama 3.3 Nemotron Super 49B V1 (thinking)","modelName":"Llama 3.3 Nemotron Super 49B V1","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":17,"score":130.78,"scoreLo":127.88,"scoreHi":133.48,"reasoningOffset":0.49,"rankEligible":true,"rankMean":364.415,"rankLo":335.0,"rankHi":396.025},{"stateId":"llama-3-70b::default","configSlug":"llama-3-70b","modelId":"llama-3-70b","modelSlug":"llama-3-70b","name":"Llama 3 70B","modelName":"Llama 3 70B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":39,"score":114.42,"scoreLo":110.21,"scoreHi":117.94,"reasoningOffset":0.0,"rankEligible":true,"rankMean":568.8808,"rankLo":542.0,"rankHi":597.0},{"stateId":"llama-3-70b-instruct::default","configSlug":"llama-3-70b-instruct","modelId":"llama-3-70b-instruct","modelSlug":"llama-3-70b-instruct","name":"Llama 3 70B Instruct","modelName":"Llama 3 70B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":61,"score":117.72,"scoreLo":114.32,"scoreHi":120.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":530.585,"rankLo":513.0,"rankHi":548.0},{"stateId":"llama-3-8b::default","configSlug":"llama-3-8b","modelId":"llama-3-8b","modelSlug":"llama-3-8b","name":"Llama 3 8B","modelName":"Llama 3 8B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":42,"score":101.72,"scoreLo":96.05,"scoreHi":106.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":705.1475,"rankLo":691.0,"rankHi":719.0},{"stateId":"llama-3-8b-instruct::default","configSlug":"llama-3-8b-instruct","modelId":"llama-3-8b-instruct","modelSlug":"llama-3-8b-instruct","name":"Llama 3 8B Instruct","modelName":"Llama 3 8B Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":56,"score":103.65,"scoreLo":98.36,"scoreHi":108.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":688.8244,"rankLo":672.975,"rankHi":703.0},{"stateId":"llama-33b::default","configSlug":"llama-33b","modelId":"llama-33b","modelSlug":"llama-33b","name":"LLaMA 33B","modelName":"LLaMA 33B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":95.39,"scoreLo":87.86,"scoreHi":101.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":754.9586,"rankLo":724.0,"rankHi":783.0},{"stateId":"llama-4-maverick-17b-128e::default","configSlug":"llama-4-maverick-17b-128e","modelId":"llama-4-maverick-17b-128e","modelSlug":"llama-4-maverick-17b-128e","name":"Llama 4 Maverick","modelName":"Llama 4 Maverick","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":122.56,"scoreLo":118.12,"scoreHi":126.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":468.005,"rankLo":420.975,"rankHi":518.0},{"stateId":"llama-4-maverick-17b-128e-instruct::default","configSlug":"llama-4-maverick-17b-128e-instruct","modelId":"llama-4-maverick-17b-128e-instruct","modelSlug":"llama-4-maverick-17b-128e-instruct","name":"Llama 4 Maverick Instruct","modelName":"Llama 4 Maverick Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":105,"score":129.5,"scoreLo":127.48,"scoreHi":131.45,"reasoningOffset":0.0,"rankEligible":true,"rankMean":382.0581,"rankLo":363.0,"rankHi":400.0},{"stateId":"llama-4-maverick-17b-128e-instruct-fp8::default","configSlug":"llama-4-maverick-17b-128e-instruct-fp8","modelId":"llama-4-maverick-17b-128e-instruct-fp8","modelSlug":"llama-4-maverick-17b-128e-instruct-fp8","name":"Llama 4 Maverick Instruct FP8","modelName":"Llama 4 Maverick Instruct FP8","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":134.38,"scoreLo":130.85,"scoreHi":137.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":316.7464,"rankLo":275.975,"rankHi":358.0},{"stateId":"llama-4-scout-17b-16e::default","configSlug":"llama-4-scout-17b-16e","modelId":"llama-4-scout-17b-16e","modelSlug":"llama-4-scout-17b-16e","name":"Llama 4 Scout","modelName":"Llama 4 Scout","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":114.68,"scoreLo":109.69,"scoreHi":119.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":566.0522,"rankLo":526.0,"rankHi":609.0},{"stateId":"llama-4-scout-17b-16e-instruct::default","configSlug":"llama-4-scout-17b-16e-instruct","modelId":"llama-4-scout-17b-16e-instruct","modelSlug":"llama-4-scout-17b-16e-instruct","name":"Llama 4 Scout Instruct","modelName":"Llama 4 Scout Instruct","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":87,"score":125.29,"scoreLo":122.68,"scoreHi":127.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":431.7789,"rankLo":414.0,"rankHi":452.0},{"stateId":"llama-65b::default","configSlug":"llama-65b","modelId":"llama-65b","modelSlug":"llama-65b","name":"LLaMA 65B","modelName":"LLaMA 65B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":98.27,"scoreLo":91.47,"scoreHi":104.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":732.1911,"rankLo":705.0,"rankHi":761.025},{"stateId":"llama-7b::default","configSlug":"llama-7b","modelId":"llama-7b","modelSlug":"llama-7b","name":"LLaMA 7B","modelName":"LLaMA 7B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":73.44,"scoreLo":62.17,"scoreHi":82.69,"reasoningOffset":0.0,"rankEligible":true,"rankMean":852.7453,"rankLo":839.0,"rankHi":863.0},{"stateId":"llava-onevision-qwen2-72b-ov-sft::default","configSlug":"llava-onevision-qwen2-72b-ov-sft","modelId":"llava-onevision-qwen2-72b-ov-sft","modelSlug":"llava-onevision-qwen2-72b-ov-sft","name":"LLaVA-OneVision 72B","modelName":"LLaVA-OneVision 72B","organizationSlug":"llava-authors","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":118.65,"scoreLo":113.1,"scoreHi":123.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":518.6339,"rankLo":458.0,"rankHi":573.0},{"stateId":"llava-v1-5-7b::default","configSlug":"llava-v1-5-7b","modelId":"llava-v1-5-7b","modelSlug":"llava-v1-5-7b","name":"LLaVA v1.5 7B","modelName":"LLaVA v1.5 7B","organizationSlug":"llava-authors","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":81.51,"scoreLo":68.48,"scoreHi":92.09,"reasoningOffset":0.0,"rankEligible":true,"rankMean":826.6242,"rankLo":790.0,"rankHi":857.0},{"stateId":"longcat-2-0::default","configSlug":"longcat-2-0","modelId":"longcat-2-0","modelSlug":"longcat-2-0","name":"LongCat-2.0","modelName":"LongCat-2.0","organizationSlug":"meituan","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":146.37,"scoreLo":143.41,"scoreHi":149.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":171.3269,"rankLo":146.0,"rankHi":198.0},{"stateId":"longcat-2-0::enabled","configSlug":"longcat-2-0-thinking-ale-bench-self-refine-x1","modelId":"longcat-2-0","modelSlug":"longcat-2-0","name":"LongCat-2.0 (thinking)","modelName":"LongCat-2.0","organizationSlug":"meituan","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":147.39,"scoreLo":143.25,"scoreHi":151.77,"reasoningOffset":1.02,"rankEligible":false},{"stateId":"longcat-2-5::enabled","configSlug":"longcat-2-5-thinking","modelId":"longcat-2-5","modelSlug":"longcat-2-5","name":"LongCat-2.5 (thinking)","modelName":"LongCat-2.5","organizationSlug":"meituan","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":13,"score":163.25,"scoreLo":159.51,"scoreHi":167.7,"reasoningOffset":1.07,"rankEligible":true,"rankMean":20.7269,"rankLo":15.0,"rankHi":31.0},{"stateId":"longcat-flash-lite::default","configSlug":"longcat-flash-lite","modelId":"longcat-flash-lite","modelSlug":"longcat-flash-lite","name":"LongCat-Flash-Lite","modelName":"LongCat-Flash-Lite","organizationSlug":"meituan","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":132.86,"scoreLo":125.23,"scoreHi":139.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":336.3386,"rankLo":255.0,"rankHi":429.025},{"stateId":"macaron-v1-venti::default","configSlug":"macaron-v1-venti-mol-router","modelId":"macaron-v1-venti","modelSlug":"macaron-v1-venti","name":"Macaron-V1-Venti","modelName":"Macaron-V1-Venti","organizationSlug":"mindlab-research","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":4,"score":162.23,"scoreLo":156.01,"scoreHi":169.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":32.4064,"rankLo":4.0,"rankHi":73.0},{"stateId":"magistral-medium-2506::default","configSlug":"magistral-medium-2506","modelId":"magistral-medium-2506","modelSlug":"magistral-medium-2506","name":"Magistral Medium 1.0","modelName":"Magistral Medium 1.0","organizationSlug":"mistral-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":130.47,"scoreLo":127.34,"scoreHi":133.25,"reasoningOffset":2.4,"rankEligible":true,"rankMean":368.6769,"rankLo":335.0,"rankHi":404.0},{"stateId":"magistral-medium-2509::default","configSlug":"magistral-medium-2509","modelId":"magistral-medium-2509","modelSlug":"magistral-medium-2509","name":"Magistral Medium 1.2","modelName":"Magistral Medium 1.2","organizationSlug":"mistral-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":135.66,"scoreLo":133.12,"scoreHi":138.23,"reasoningOffset":2.4,"rankEligible":true,"rankMean":299.0139,"rankLo":269.0,"rankHi":330.0},{"stateId":"magistral-small-2506::default","configSlug":"magistral-small-2506","modelId":"magistral-small-2506","modelSlug":"magistral-small-2506","name":"Magistral Small 1.0","modelName":"Magistral Small 1.0","organizationSlug":"mistral-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":129.02,"scoreLo":125.38,"scoreHi":132.42,"reasoningOffset":2.4,"rankEligible":true,"rankMean":387.4236,"rankLo":349.0,"rankHi":423.0},{"stateId":"magistral-small-2509::default","configSlug":"magistral-small-2509","modelId":"magistral-small-2509","modelSlug":"magistral-small-2509","name":"Magistral Small 1.2","modelName":"Magistral Small 1.2","organizationSlug":"mistral-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":130.55,"scoreLo":127.84,"scoreHi":133.21,"reasoningOffset":2.4,"rankEligible":true,"rankMean":367.5006,"rankLo":339.0,"rankHi":398.0},{"stateId":"marin-8b-instruct::default","configSlug":"marin-8b-instruct-helm-capabilities","modelId":"marin-8b-instruct","modelSlug":"marin-8b-instruct","name":"Marin 8B Instruct","modelName":"Marin 8B Instruct","organizationSlug":"marin","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":5,"score":74.53,"scoreLo":61.31,"scoreHi":85.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":848.9686,"rankLo":824.0,"rankHi":866.0},{"stateId":"medgemma-1-5-4b-it::default","configSlug":"medgemma-1-5-4b-it","modelId":"medgemma-1-5-4b-it","modelSlug":"medgemma-1-5-4b-it","name":"MedGemma 1.5 4B IT","modelName":"MedGemma 1.5 4B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":117.94,"scoreLo":106.95,"scoreHi":127.31,"reasoningOffset":0.0,"rankEligible":true,"rankMean":525.8183,"rankLo":411.0,"rankHi":650.025},{"stateId":"medgemma-27b-it::default","configSlug":"medgemma-27b-it","modelId":"medgemma-27b-it","modelSlug":"medgemma-27b-it","name":"MedGemma 27B IT","modelName":"MedGemma 27B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":133.03,"scoreLo":124.68,"scoreHi":140.77,"reasoningOffset":0.0,"rankEligible":true,"rankMean":334.0025,"rankLo":229.0,"rankHi":440.0},{"stateId":"medgemma-27b-text-it::default","configSlug":"medgemma-27b-text-it","modelId":"medgemma-27b-text-it","modelSlug":"medgemma-27b-text-it","name":"MedGemma 27B Text IT","modelName":"MedGemma 27B Text IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":126.47,"scoreLo":120.43,"scoreHi":132.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":418.3025,"rankLo":345.0,"rankHi":494.0},{"stateId":"medgemma-4b-it::default","configSlug":"medgemma-4b-it","modelId":"medgemma-4b-it","modelSlug":"medgemma-4b-it","name":"MedGemma 4B IT","modelName":"MedGemma 4B IT","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":107.81,"scoreLo":100.83,"scoreHi":114.11,"reasoningOffset":0.0,"rankEligible":true,"rankMean":646.8611,"rankLo":583.0,"rankHi":701.0},{"stateId":"mercury-2::default","configSlug":"mercury-2","modelId":"mercury-2","modelSlug":"mercury-2","name":"Mercury 2","modelName":"Mercury 2","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":19,"score":139.68,"scoreLo":136.91,"scoreHi":142.55,"reasoningOffset":0.0,"rankEligible":true,"rankMean":244.4375,"rankLo":215.0,"rankHi":278.0},{"stateId":"mercury-2-5::default","configSlug":"mercury-2-5","modelId":"mercury-2-5","modelSlug":"mercury-2-5","name":"Mercury 2.5","modelName":"Mercury 2.5","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":1,"score":138.67,"scoreLo":135.75,"scoreHi":141.52,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"mercury-2-5::effort:high","configSlug":"mercury-2-5-high","modelId":"mercury-2-5","modelSlug":"mercury-2-5","name":"Mercury 2.5 (high)","modelName":"Mercury 2.5","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":19,"score":140.2,"scoreLo":137.28,"scoreHi":143.1,"reasoningOffset":1.53,"rankEligible":true},{"stateId":"mercury-2-5::enabled","configSlug":"mercury-2-5-thinking","modelId":"mercury-2-5","modelSlug":"mercury-2-5","name":"Mercury 2.5 (thinking)","modelName":"Mercury 2.5","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":141.08,"scoreLo":137.71,"scoreHi":144.59,"reasoningOffset":2.41,"rankEligible":true,"rankMean":227.4853,"rankLo":195.0,"rankHi":268.0},{"stateId":"mercury-2::effort:high","configSlug":"mercury-2-high","modelId":"mercury-2","modelSlug":"mercury-2","name":"Mercury 2 (high)","modelName":"Mercury 2","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":140.34,"scoreLo":136.59,"scoreHi":144.18,"reasoningOffset":0.66,"rankEligible":false},{"stateId":"mercury-2::effort:minimal","configSlug":"mercury-2-minimal","modelId":"mercury-2","modelSlug":"mercury-2","name":"Mercury 2 (minimal)","modelName":"Mercury 2","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":135.02,"scoreLo":129.89,"scoreHi":140.19,"reasoningOffset":-4.66,"rankEligible":false},{"stateId":"mercury-2::enabled","configSlug":"mercury-2-thinking-ale-bench-self-refine-x1","modelId":"mercury-2","modelSlug":"mercury-2","name":"Mercury 2 (thinking)","modelName":"Mercury 2","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":141.12,"scoreLo":137.22,"scoreHi":145.14,"reasoningOffset":1.44,"rankEligible":false},{"stateId":"mercury-3::default","configSlug":"mercury-3","modelId":"mercury-3","modelSlug":"mercury-3","name":"Mercury 3","modelName":"Mercury 3","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":155.79,"scoreLo":152.6,"scoreHi":159.51,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"mercury-3::enabled","configSlug":"mercury-3-concept-synth-cascade","modelId":"mercury-3","modelSlug":"mercury-3","name":"Mercury 3 (enabled)","modelName":"Mercury 3","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":156.65,"scoreLo":152.74,"scoreHi":161.13,"reasoningOffset":0.86,"rankEligible":false},{"stateId":"mercury-3::effort:high","configSlug":"mercury-3-high","modelId":"mercury-3","modelSlug":"mercury-3","name":"Mercury 3 (high)","modelName":"Mercury 3","organizationSlug":"inception-labs","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":14,"score":157.15,"scoreLo":153.9,"scoreHi":160.98,"reasoningOffset":1.36,"rankEligible":true,"rankMean":73.2189,"rankLo":62.0,"rankHi":82.0},{"stateId":"mimo-v2-5::default","configSlug":"mimo-v2-5","modelId":"mimo-v2-5","modelSlug":"mimo-v2-5","name":"MiMo-V2.5","modelName":"MiMo-V2.5","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":38,"score":150.17,"scoreLo":147.72,"scoreHi":152.92,"reasoningOffset":0.0,"rankEligible":true,"rankMean":129.7853,"rankLo":116.0,"rankHi":144.0},{"stateId":"mimo-v2-5::effort:high","configSlug":"mimo-v2-5-high","modelId":"mimo-v2-5","modelSlug":"mimo-v2-5","name":"MiMo-V2.5 (high)","modelName":"MiMo-V2.5","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":150.08,"scoreLo":146.84,"scoreHi":153.73,"reasoningOffset":-0.09,"rankEligible":false},{"stateId":"mimo-v2-5-pro::default","configSlug":"mimo-v2-5-pro","modelId":"mimo-v2-5-pro","modelSlug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro","modelName":"MiMo-V2.5-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":37,"score":151.12,"scoreLo":148.57,"scoreHi":154.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":119.3992,"rankLo":107.0,"rankHi":134.0},{"stateId":"mimo-v2-5-pro::effort:high","configSlug":"mimo-v2-5-pro-high","modelId":"mimo-v2-5-pro","modelSlug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro (high)","modelName":"MiMo-V2.5-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":152.25,"scoreLo":149.15,"scoreHi":155.86,"reasoningOffset":1.13,"rankEligible":false},{"stateId":"mimo-v2-5-pro::disabled","configSlug":"mimo-v2-5-pro-no-thinking","modelId":"mimo-v2-5-pro","modelSlug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro (disabled)","modelName":"MiMo-V2.5-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":145.12,"scoreLo":141.14,"scoreHi":149.1,"reasoningOffset":-6.0,"rankEligible":true},{"stateId":"mimo-v2-5-pro::enabled","configSlug":"mimo-v2-5-pro-thinking","modelId":"mimo-v2-5-pro","modelSlug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro (thinking)","modelName":"MiMo-V2.5-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":21,"score":151.08,"scoreLo":148.43,"scoreHi":154.26,"reasoningOffset":-0.05,"rankEligible":true},{"stateId":"mimo-v2-5-pro::effort:xhigh","configSlug":"mimo-v2-5-pro-xhigh-genebench-tools","modelId":"mimo-v2-5-pro","modelSlug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro (xhigh)","modelName":"MiMo-V2.5-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":151.97,"scoreLo":148.56,"scoreHi":155.76,"reasoningOffset":0.85,"rankEligible":false},{"stateId":"mimo-v2-5::enabled","configSlug":"mimo-v2-5-thinking","modelId":"mimo-v2-5","modelSlug":"mimo-v2-5","name":"MiMo-V2.5 (thinking)","modelName":"MiMo-V2.5","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":16,"score":150.1,"scoreLo":147.37,"scoreHi":153.15,"reasoningOffset":-0.07,"rankEligible":true},{"stateId":"mimo-v2-5::effort:xhigh","configSlug":"mimo-v2-5-xhigh-genebench-tools","modelId":"mimo-v2-5","modelSlug":"mimo-v2-5","name":"MiMo-V2.5 (xhigh)","modelName":"MiMo-V2.5","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":150.89,"scoreLo":147.54,"scoreHi":154.52,"reasoningOffset":0.72,"rankEligible":false},{"stateId":"mimo-v2-6-flash::default","configSlug":"mimo-v2-6-flash","modelId":"mimo-v2-6-flash","modelSlug":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash","modelName":"MiMo-V2.6-Flash","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":41,"score":159.92,"scoreLo":156.7,"scoreHi":163.81,"reasoningOffset":0.0,"rankEligible":true,"rankMean":49.7075,"rankLo":40.0,"rankHi":60.0},{"stateId":"mimo-v2-6-flash::effort:high","configSlug":"mimo-v2-6-flash-high","modelId":"mimo-v2-6-flash","modelSlug":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash (high)","modelName":"MiMo-V2.6-Flash","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":159.68,"scoreLo":155.93,"scoreHi":164.13,"reasoningOffset":-0.25,"rankEligible":false},{"stateId":"mimo-v2-6-flash::enabled","configSlug":"mimo-v2-6-flash-thinking","modelId":"mimo-v2-6-flash","modelSlug":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash (thinking)","modelName":"MiMo-V2.6-Flash","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":13,"score":159.19,"scoreLo":155.76,"scoreHi":163.27,"reasoningOffset":-0.74,"rankEligible":true},{"stateId":"mimo-v2-6-pro::default","configSlug":"mimo-v2-6-pro","modelId":"mimo-v2-6-pro","modelSlug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","modelName":"MiMo-V2.6-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":50,"score":160.63,"scoreLo":157.25,"scoreHi":164.62,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mimo-v2-6-pro::effort:high","configSlug":"mimo-v2-6-pro-high","modelId":"mimo-v2-6-pro","modelSlug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro (high)","modelName":"MiMo-V2.6-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":4,"score":160.59,"scoreLo":156.9,"scoreHi":164.89,"reasoningOffset":-0.04,"rankEligible":true},{"stateId":"mimo-v2-6-pro::enabled","configSlug":"mimo-v2-6-pro-thinking","modelId":"mimo-v2-6-pro","modelSlug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro (thinking)","modelName":"MiMo-V2.6-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":17,"score":161.59,"scoreLo":158.11,"scoreHi":165.88,"reasoningOffset":0.96,"rankEligible":true,"rankMean":32.1519,"rankLo":22.0,"rankHi":45.0},{"stateId":"mimo-v2-flash::default","configSlug":"mimo-v2-flash","modelId":"mimo-v2-flash","modelSlug":"mimo-v2-flash","name":"MiMo-V2-Flash","modelName":"MiMo-V2-Flash","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":144.02,"scoreLo":141.01,"scoreHi":147.18,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mimo-v2-flash::effort:high","configSlug":"mimo-v2-flash-high","modelId":"mimo-v2-flash","modelSlug":"mimo-v2-flash","name":"MiMo-V2-Flash (high)","modelName":"MiMo-V2-Flash","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":144.92,"scoreLo":141.05,"scoreHi":148.83,"reasoningOffset":0.9,"rankEligible":false},{"stateId":"mimo-v2-flash::disabled","configSlug":"mimo-v2-flash-no-thinking","modelId":"mimo-v2-flash","modelSlug":"mimo-v2-flash","name":"MiMo-V2-Flash (disabled)","modelName":"MiMo-V2-Flash","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":13,"score":137.76,"scoreLo":134.66,"scoreHi":140.8,"reasoningOffset":-6.26,"rankEligible":true},{"stateId":"mimo-v2-flash::enabled","configSlug":"mimo-v2-flash-thinking","modelId":"mimo-v2-flash","modelSlug":"mimo-v2-flash","name":"MiMo-V2-Flash (thinking)","modelName":"MiMo-V2-Flash","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":146.45,"scoreLo":143.39,"scoreHi":149.8,"reasoningOffset":2.43,"rankEligible":true,"rankMean":170.4128,"rankLo":142.0,"rankHi":198.0},{"stateId":"mimo-v2-omni::default","configSlug":"mimo-v2-omni","modelId":"mimo-v2-omni","modelSlug":"mimo-v2-omni","name":"MiMo-V2-Omni","modelName":"MiMo-V2-Omni","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":146.67,"scoreLo":142.65,"scoreHi":150.98,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mimo-v2-omni::effort:minimal","configSlug":"mimo-v2-omni-minimal","modelId":"mimo-v2-omni","modelSlug":"mimo-v2-omni","name":"MiMo-V2-Omni (minimal)","modelName":"MiMo-V2-Omni","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":142.56,"scoreLo":136.86,"scoreHi":148.41,"reasoningOffset":-4.12,"rankEligible":false},{"stateId":"mimo-v2-pro::default","configSlug":"mimo-v2-pro","modelId":"mimo-v2-pro","modelSlug":"mimo-v2-pro","name":"MiMo-V2-Pro","modelName":"MiMo-V2-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":146.24,"scoreLo":143.4,"scoreHi":149.35,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mimo-v2-pro::effort:high","configSlug":"mimo-v2-pro-high","modelId":"mimo-v2-pro","modelSlug":"mimo-v2-pro","name":"MiMo-V2-Pro (high)","modelName":"MiMo-V2-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":146.95,"scoreLo":143.32,"scoreHi":150.73,"reasoningOffset":0.71,"rankEligible":false},{"stateId":"mimo-v2-pro::effort:minimal","configSlug":"mimo-v2-pro-minimal","modelId":"mimo-v2-pro","modelSlug":"mimo-v2-pro","name":"MiMo-V2-Pro (minimal)","modelName":"MiMo-V2-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":141.94,"scoreLo":136.95,"scoreHi":147.11,"reasoningOffset":-4.3,"rankEligible":false},{"stateId":"minicpm-o-2-6::default","configSlug":"minicpm-o-2-6","modelId":"minicpm-o-2-6","modelSlug":"minicpm-o-2-6","name":"MiniCPM-o 2.6","modelName":"MiniCPM-o 2.6","organizationSlug":"openbmb","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":115.32,"scoreLo":109.82,"scoreHi":120.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":558.3722,"rankLo":511.0,"rankHi":610.025},{"stateId":"minicpm-v-2-6::default","configSlug":"minicpm-v-2-6","modelId":"minicpm-v-2-6","modelSlug":"minicpm-v-2-6","name":"MiniCPM-V 2.6","modelName":"MiniCPM-V 2.6","organizationSlug":"openbmb","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":111.72,"scoreLo":106.03,"scoreHi":116.73,"reasoningOffset":0.0,"rankEligible":true,"rankMean":602.1717,"rankLo":554.0,"rankHi":649.0},{"stateId":"minicpm-v-4-6::default","configSlug":"minicpm-v-4-6","modelId":"minicpm-v-4-6","modelSlug":"minicpm-v-4-6","name":"MiniCPM-V 4.6","modelName":"MiniCPM-V 4.6","organizationSlug":"openbmb","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":115.33,"scoreLo":107.13,"scoreHi":122.22,"reasoningOffset":0.0,"rankEligible":true,"rankMean":558.435,"rankLo":478.0,"rankHi":647.0},{"stateId":"minicpm5-1b::disabled","configSlug":"minicpm5-1b-no-thinking","modelId":"minicpm5-1b","modelSlug":"minicpm5-1b","name":"MiniCPM5-1B (disabled)","modelName":"MiniCPM5-1B","organizationSlug":"openbmb","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":5,"score":114.58,"scoreLo":105.99,"scoreHi":122.26,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"minicpm5-1b::enabled","configSlug":"minicpm5-1b-thinking","modelId":"minicpm5-1b","modelSlug":"minicpm5-1b","name":"MiniCPM5-1B (thinking)","modelName":"MiniCPM5-1B","organizationSlug":"openbmb","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":119.44,"scoreLo":111.33,"scoreHi":126.43,"reasoningOffset":0.87,"rankEligible":true,"rankMean":507.9011,"rankLo":421.0,"rankHi":600.0},{"stateId":"minicpm5-2b::enabled","configSlug":"minicpm5-2b-thinking","modelId":"minicpm5-2b","modelSlug":"minicpm5-2b","name":"MiniCPM5-2B (thinking)","modelName":"MiniCPM5-2B","organizationSlug":"openbmb","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":136.96,"scoreLo":133.17,"scoreHi":140.59,"reasoningOffset":1.07,"rankEligible":true,"rankMean":281.3669,"rankLo":234.0,"rankHi":332.0},{"stateId":"minimax-m1-40k::effort:40k","configSlug":"minimax-m1-40k","modelId":"minimax-m1-40k","modelSlug":"minimax-m1-40k","name":"MiniMax M1 40k (40k)","modelName":"MiniMax M1 40k","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"40k","reasoningCondition":"effort:40k","isDefault":true,"n":5,"score":132.92,"scoreLo":127.66,"scoreHi":137.62,"reasoningOffset":2.09,"rankEligible":true,"rankMean":335.9994,"rankLo":276.0,"rankHi":401.0},{"stateId":"minimax-m1-80k::effort:80k","configSlug":"minimax-m1-80k","modelId":"minimax-m1-80k","modelSlug":"minimax-m1-80k","name":"MiniMax M1 80k (80k)","modelName":"MiniMax M1 80k","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"80k","reasoningCondition":"effort:80k","isDefault":true,"n":7,"score":135.58,"scoreLo":131.04,"scoreHi":139.86,"reasoningOffset":2.57,"rankEligible":true,"rankMean":300.1161,"rankLo":245.0,"rankHi":356.0},{"stateId":"minimax-m2::default","configSlug":"minimax-m2","modelId":"minimax-m2","modelSlug":"minimax-m2","name":"MiniMax M2","modelName":"MiniMax M2","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":31,"score":143.4,"scoreLo":141.11,"scoreHi":145.81,"reasoningOffset":2.4,"rankEligible":true,"rankMean":202.7456,"rankLo":185.0,"rankHi":219.0},{"stateId":"minimax-m2-1::default","configSlug":"minimax-m2-1","modelId":"minimax-m2-1","modelSlug":"minimax-m2-1","name":"MiniMax M2.1","modelName":"MiniMax M2.1","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":35,"score":144.3,"scoreLo":141.97,"scoreHi":146.86,"reasoningOffset":2.4,"rankEligible":true,"rankMean":193.4181,"rankLo":174.0,"rankHi":211.0},{"stateId":"minimax-m2-1::effort:high","configSlug":"minimax-m2-1-high","modelId":"minimax-m2-1","modelSlug":"minimax-m2-1","name":"MiniMax M2.1 (high)","modelName":"MiniMax M2.1","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":141.94,"scoreLo":138.69,"scoreHi":145.41,"reasoningOffset":0.03,"rankEligible":false},{"stateId":"minimax-m2-5::default","configSlug":"minimax-m2-5","modelId":"minimax-m2-5","modelSlug":"minimax-m2-5","name":"MiniMax M2.5","modelName":"MiniMax M2.5","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":44,"score":145.75,"scoreLo":143.62,"scoreHi":148.11,"reasoningOffset":2.4,"rankEligible":true,"rankMean":178.1172,"rankLo":165.0,"rankHi":192.0},{"stateId":"minimax-m2-5::effort:high","configSlug":"minimax-m2-5-high","modelId":"minimax-m2-5","modelSlug":"minimax-m2-5","name":"MiniMax M2.5 (high)","modelName":"MiniMax M2.5","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":143.18,"scoreLo":140.2,"scoreHi":146.4,"reasoningOffset":-0.17,"rankEligible":false},{"stateId":"minimax-m2-5::effort:minimal","configSlug":"minimax-m2-5-minimal","modelId":"minimax-m2-5","modelSlug":"minimax-m2-5","name":"MiniMax M2.5 (minimal)","modelName":"MiniMax M2.5","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":138.52,"scoreLo":133.94,"scoreHi":143.11,"reasoningOffset":-4.82,"rankEligible":false},{"stateId":"minimax-m2-5::effort:xhigh","configSlug":"minimax-m2-5-xhigh","modelId":"minimax-m2-5","modelSlug":"minimax-m2-5","name":"MiniMax M2.5 (xhigh)","modelName":"MiniMax M2.5","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":144.19,"scoreLo":141.31,"scoreHi":147.36,"reasoningOffset":0.85,"rankEligible":false},{"stateId":"minimax-m2-7::default","configSlug":"minimax-m2-7","modelId":"minimax-m2-7","modelSlug":"minimax-m2-7","name":"MiniMax M2.7","modelName":"MiniMax M2.7","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":76,"score":147.16,"scoreLo":145.12,"scoreHi":149.54,"reasoningOffset":2.4,"rankEligible":true,"rankMean":163.0692,"rankLo":152.0,"rankHi":174.0},{"stateId":"minimax-m2-7::effort:high","configSlug":"minimax-m2-7-high","modelId":"minimax-m2-7","modelSlug":"minimax-m2-7","name":"MiniMax M2.7 (high)","modelName":"MiniMax M2.7","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":22,"score":145.28,"scoreLo":143.01,"scoreHi":147.76,"reasoningOffset":0.51,"rankEligible":true},{"stateId":"minimax-m2-7::effort:minimal","configSlug":"minimax-m2-7-minimal","modelId":"minimax-m2-7","modelSlug":"minimax-m2-7","name":"MiniMax M2.7 (minimal)","modelName":"MiniMax M2.7","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":140.23,"scoreLo":135.7,"scoreHi":144.92,"reasoningOffset":-4.53,"rankEligible":false},{"stateId":"minimax-m2-7::effort:xhigh","configSlug":"minimax-m2-7-xhigh","modelId":"minimax-m2-7","modelSlug":"minimax-m2-7","name":"MiniMax M2.7 (xhigh)","modelName":"MiniMax M2.7","organizationSlug":"minimax","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":145.99,"scoreLo":142.96,"scoreHi":149.21,"reasoningOffset":1.23,"rankEligible":false},{"stateId":"minimax-m3::default","configSlug":"minimax-m3","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":90,"score":152.87,"scoreLo":150.31,"scoreHi":155.74,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"minimax-m3-1-flash::default","configSlug":"minimax-m3-1-flash","modelId":"minimax-m3-1-flash","modelSlug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","modelName":"MiniMax M3.1 Flash","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":160.8,"scoreLo":157.43,"scoreHi":164.79,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"minimax-m3-1-flash::effort:high","configSlug":"minimax-m3-1-flash-high-claude-code","modelId":"minimax-m3-1-flash","modelSlug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash (high)","modelName":"MiniMax M3.1 Flash","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":160.28,"scoreLo":156.5,"scoreHi":164.63,"reasoningOffset":-0.52,"rankEligible":false},{"stateId":"minimax-m3-1-flash::effort:max","configSlug":"minimax-m3-1-flash-max","modelId":"minimax-m3-1-flash","modelSlug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash (max)","modelName":"MiniMax M3.1 Flash","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":19,"score":160.97,"scoreLo":157.4,"scoreHi":165.33,"reasoningOffset":0.18,"rankEligible":true,"rankMean":38.5614,"rankLo":24.0,"rankHi":54.0},{"stateId":"minimax-m3-1-flash::effort:medium","configSlug":"minimax-m3-1-flash-medium","modelId":"minimax-m3-1-flash","modelSlug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash (medium)","modelName":"MiniMax M3.1 Flash","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":159.6,"scoreLo":155.0,"scoreHi":164.91,"reasoningOffset":-1.2,"rankEligible":false},{"stateId":"minimax-m3-1-flash::effort:xhigh","configSlug":"minimax-m3-1-flash-xhigh-mini-swe-agent","modelId":"minimax-m3-1-flash","modelSlug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash (xhigh)","modelName":"MiniMax M3.1 Flash","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":161.94,"scoreLo":157.86,"scoreHi":166.83,"reasoningOffset":1.14,"rankEligible":false},{"stateId":"minimax-m3::effort:adaptive","configSlug":"minimax-m3-adaptive-mini-swe-agent","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (adaptive)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"adaptive","reasoningCondition":"effort:adaptive","isDefault":false,"n":1,"score":153.08,"scoreLo":147.49,"scoreHi":160.23,"reasoningOffset":0.21,"rankEligible":false},{"stateId":"minimax-m3::effort:high","configSlug":"minimax-m3-high","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (high)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":34,"score":152.32,"scoreLo":149.76,"scoreHi":155.25,"reasoningOffset":-0.55,"rankEligible":true},{"stateId":"minimax-m3::effort:max","configSlug":"minimax-m3-max","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (max)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":2,"score":153.6,"scoreLo":150.44,"scoreHi":157.26,"reasoningOffset":0.74,"rankEligible":false},{"stateId":"minimax-m3::effort:medium","configSlug":"minimax-m3-medium","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (medium)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":4,"score":150.97,"scoreLo":147.62,"scoreHi":154.79,"reasoningOffset":-1.89,"rankEligible":true},{"stateId":"minimax-m3::effort:minimal","configSlug":"minimax-m3-minimal","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (minimal)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":148.45,"scoreLo":143.77,"scoreHi":153.56,"reasoningOffset":-4.42,"rankEligible":false},{"stateId":"minimax-m3::disabled","configSlug":"minimax-m3-no-thinking","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (disabled)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":145.44,"scoreLo":142.53,"scoreHi":148.55,"reasoningOffset":-7.44,"rankEligible":true},{"stateId":"minimax-m3::enabled","configSlug":"minimax-m3-thinking","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (thinking)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":29,"score":153.2,"scoreLo":150.46,"scoreHi":156.23,"reasoningOffset":0.33,"rankEligible":true,"rankMean":100.1469,"rankLo":88.975,"rankHi":111.0},{"stateId":"minimax-m3::effort:xhigh","configSlug":"minimax-m3-xhigh-genebench-tools","modelId":"minimax-m3","modelSlug":"minimax-m3","name":"MiniMax M3 (xhigh)","modelName":"MiniMax M3","organizationSlug":"minimax","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":152.62,"scoreLo":149.54,"scoreHi":156.02,"reasoningOffset":-0.25,"rankEligible":false},{"stateId":"ministral-3-14b-base-2512::default","configSlug":"ministral-3-14b-base-2512","modelId":"ministral-3-14b-base-2512","modelSlug":"ministral-3-14b-base-2512","name":"Ministral 3 14B Base 2512","modelName":"Ministral 3 14B Base 2512","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":116.63,"scoreLo":111.99,"scoreHi":120.86,"reasoningOffset":0.0,"rankEligible":true,"rankMean":542.9614,"rankLo":505.0,"rankHi":584.0},{"stateId":"ministral-3-14b-instruct-2512::default","configSlug":"ministral-3-14b-instruct-2512","modelId":"ministral-3-14b-instruct-2512","modelSlug":"ministral-3-14b-instruct-2512","name":"Ministral 3 14B Instruct 2512","modelName":"Ministral 3 14B Instruct 2512","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":128.79,"scoreLo":125.76,"scoreHi":131.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":390.6,"rankLo":359.975,"rankHi":419.0},{"stateId":"ministral-3-14b-reasoning-2512::default","configSlug":"ministral-3-14b-reasoning-2512","modelId":"ministral-3-14b-reasoning-2512","modelSlug":"ministral-3-14b-reasoning-2512","name":"Ministral 3 14B Reasoning 2512","modelName":"Ministral 3 14B Reasoning 2512","organizationSlug":"mistral-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":6,"score":136.07,"scoreLo":131.48,"scoreHi":140.76,"reasoningOffset":2.4,"rankEligible":true,"rankMean":293.4522,"rankLo":233.0,"rankHi":350.025},{"stateId":"ministral-3-3b-base-2512::default","configSlug":"ministral-3-3b-base-2512","modelId":"ministral-3-3b-base-2512","modelSlug":"ministral-3-3b-base-2512","name":"Ministral 3 3B Base 2512","modelName":"Ministral 3 3B Base 2512","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":105.67,"scoreLo":99.57,"scoreHi":111.01,"reasoningOffset":0.0,"rankEligible":true,"rankMean":669.5703,"rankLo":630.0,"rankHi":705.0},{"stateId":"ministral-3-3b-instruct-2512::default","configSlug":"ministral-3-3b-instruct-2512","modelId":"ministral-3-3b-instruct-2512","modelSlug":"ministral-3-3b-instruct-2512","name":"Ministral 3 3B Instruct 2512","modelName":"Ministral 3 3B Instruct 2512","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":118.66,"scoreLo":114.61,"scoreHi":122.25,"reasoningOffset":0.0,"rankEligible":true,"rankMean":519.3089,"rankLo":484.0,"rankHi":552.0},{"stateId":"ministral-3-3b-reasoning-2512::default","configSlug":"ministral-3-3b-reasoning-2512","modelId":"ministral-3-3b-reasoning-2512","modelSlug":"ministral-3-3b-reasoning-2512","name":"Ministral 3 3B Reasoning 2512","modelName":"Ministral 3 3B Reasoning 2512","organizationSlug":"mistral-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":7,"score":124.8,"scoreLo":119.64,"scoreHi":129.6,"reasoningOffset":2.4,"rankEligible":true,"rankMean":438.8814,"rankLo":386.0,"rankHi":500.025},{"stateId":"ministral-3-8b-base-2512::default","configSlug":"ministral-3-8b-base-2512","modelId":"ministral-3-8b-base-2512","modelSlug":"ministral-3-8b-base-2512","name":"Ministral 3 8B Base 2512","modelName":"Ministral 3 8B Base 2512","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":112.01,"scoreLo":106.62,"scoreHi":116.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":598.8994,"rankLo":556.0,"rankHi":643.0},{"stateId":"ministral-3-8b-instruct-2512::default","configSlug":"ministral-3-8b-instruct-2512","modelId":"ministral-3-8b-instruct-2512","modelSlug":"ministral-3-8b-instruct-2512","name":"Ministral 3 8B Instruct 2512","modelName":"Ministral 3 8B Instruct 2512","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":125.88,"scoreLo":122.55,"scoreHi":128.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":424.865,"rankLo":396.0,"rankHi":457.0},{"stateId":"ministral-3-8b-reasoning-2512::default","configSlug":"ministral-3-8b-reasoning-2512","modelId":"ministral-3-8b-reasoning-2512","modelSlug":"ministral-3-8b-reasoning-2512","name":"Ministral 3 8B Reasoning 2512","modelName":"Ministral 3 8B Reasoning 2512","organizationSlug":"mistral-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":7,"score":131.56,"scoreLo":127.2,"scoreHi":135.8,"reasoningOffset":2.4,"rankEligible":true,"rankMean":354.3069,"rankLo":302.0,"rankHi":405.025},{"stateId":"ministral-3b-2410::default","configSlug":"ministral-3b-2410","modelId":"ministral-3b-2410","modelSlug":"ministral-3b-2410","name":"Ministral 3B","modelName":"Ministral 3B","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":106.81,"scoreLo":97.25,"scoreHi":115.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":655.4089,"rankLo":570.0,"rankHi":731.0},{"stateId":"ministral-8b-instruct-2410::default","configSlug":"ministral-8b-instruct-2410","modelId":"ministral-8b-instruct-2410","modelSlug":"ministral-8b-instruct-2410","name":"Ministral 8B Instruct 2410","modelName":"Ministral 8B Instruct 2410","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":106.35,"scoreLo":99.87,"scoreHi":111.7,"reasoningOffset":0.0,"rankEligible":true,"rankMean":662.7719,"rankLo":620.0,"rankHi":701.0},{"stateId":"mistral-7b-instruct-v0-1::default","configSlug":"mistral-7b-instruct-v0-1","modelId":"mistral-7b-instruct-v0-1","modelSlug":"mistral-7b-instruct-v0-1","name":"Mistral 7B Instruct v0.1","modelName":"Mistral 7B Instruct v0.1","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":32,"score":92.64,"scoreLo":85.72,"scoreHi":98.43,"reasoningOffset":0.0,"rankEligible":true,"rankMean":774.5267,"rankLo":763.0,"rankHi":787.0},{"stateId":"mistral-7b-instruct-v0-2::default","configSlug":"mistral-7b-instruct-v0-2","modelId":"mistral-7b-instruct-v0-2","modelSlug":"mistral-7b-instruct-v0-2","name":"Mistral 7B Instruct v0.2","modelName":"Mistral 7B Instruct v0.2","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":97.45,"scoreLo":90.63,"scoreHi":103.02,"reasoningOffset":0.0,"rankEligible":true,"rankMean":738.7592,"rankLo":716.0,"rankHi":763.0},{"stateId":"mistral-7b-instruct-v0-3::default","configSlug":"mistral-7b-instruct-v0-3","modelId":"mistral-7b-instruct-v0-3","modelSlug":"mistral-7b-instruct-v0-3","name":"Mistral 7B Instruct v0.3","modelName":"Mistral 7B Instruct v0.3","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":96.81,"scoreLo":89.95,"scoreHi":102.75,"reasoningOffset":0.0,"rankEligible":true,"rankMean":743.9942,"rankLo":719.0,"rankHi":769.0},{"stateId":"mistral-7b-v0-1::default","configSlug":"mistral-7b-v0-1","modelId":"mistral-7b-v0-1","modelSlug":"mistral-7b-v0-1","name":"Mistral 7B v0.1 Base","modelName":"Mistral 7B v0.1 Base","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":40,"score":97.59,"scoreLo":91.44,"scoreHi":102.75,"reasoningOffset":0.0,"rankEligible":true,"rankMean":737.3797,"rankLo":724.0,"rankHi":751.0},{"stateId":"mistral-large-2402::default","configSlug":"mistral-large-2402","modelId":"mistral-large-2402","modelSlug":"mistral-large-2402","name":"Mistral Large 1.0","modelName":"Mistral Large 1.0","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":115.84,"scoreLo":111.93,"scoreHi":119.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":551.9358,"rankLo":526.0,"rankHi":580.0},{"stateId":"mistral-large-3-675b-instruct-2512::default","configSlug":"mistral-large-3-675b-instruct-2512","modelId":"mistral-large-3-675b-instruct-2512","modelSlug":"mistral-large-3-675b-instruct-2512","name":"Mistral Large 3 675B Instruct 2512","modelName":"Mistral Large 3 675B Instruct 2512","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":68,"score":134.68,"scoreLo":132.67,"scoreHi":136.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":312.5467,"rankLo":291.0,"rankHi":335.025},{"stateId":"mistral-large-4::default","configSlug":"mistral-large-4","modelId":"mistral-large-4","modelSlug":"mistral-large-4","name":"Mistral Large 4 Preview","modelName":"Mistral Large 4 Preview","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":159.04,"scoreLo":155.64,"scoreHi":162.95,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mistral-large-4-1t-a52b::effort:high","configSlug":"mistral-large-4-1t-a52b-high","modelId":"mistral-large-4-1t-a52b","modelSlug":"mistral-large-4-1t-a52b","name":"Mistral Large 4 (high)","modelName":"Mistral Large 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":17,"score":159.88,"scoreLo":156.49,"scoreHi":163.92,"reasoningOffset":0.73,"rankEligible":true,"rankMean":50.1956,"rankLo":37.0,"rankHi":64.0},{"stateId":"mistral-large-4-1t-a52b::effort:low","configSlug":"mistral-large-4-1t-a52b-low-mistral-vibe","modelId":"mistral-large-4-1t-a52b","modelSlug":"mistral-large-4-1t-a52b","name":"Mistral Large 4 (low)","modelName":"Mistral Large 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":156.74,"scoreLo":152.56,"scoreHi":161.3,"reasoningOffset":-2.41,"rankEligible":false},{"stateId":"mistral-large-4-1t-a52b::effort:medium","configSlug":"mistral-large-4-1t-a52b-medium","modelId":"mistral-large-4-1t-a52b","modelSlug":"mistral-large-4-1t-a52b","name":"Mistral Large 4 (medium)","modelName":"Mistral Large 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":8,"score":158.44,"scoreLo":155.06,"scoreHi":162.39,"reasoningOffset":-0.7,"rankEligible":true},{"stateId":"mistral-large-4-1t-a52b::default","configSlug":"mistral-large-4-1t-a52b-pi","modelId":"mistral-large-4-1t-a52b","modelSlug":"mistral-large-4-1t-a52b","name":"Mistral Large 4","modelName":"Mistral Large 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":13,"score":159.15,"scoreLo":155.82,"scoreHi":163.13,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mistral-large-4::effort:high","configSlug":"mistral-large-4-high","modelId":"mistral-large-4","modelSlug":"mistral-large-4","name":"Mistral Large 4 Preview (high)","modelName":"Mistral Large 4 Preview","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":47,"score":156.72,"scoreLo":153.72,"scoreHi":160.24,"reasoningOffset":-2.32,"rankEligible":true},{"stateId":"mistral-large-4::disabled","configSlug":"mistral-large-4-no-thinking","modelId":"mistral-large-4","modelSlug":"mistral-large-4","name":"Mistral Large 4 Preview (disabled)","modelName":"Mistral Large 4 Preview","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":153.73,"scoreLo":146.75,"scoreHi":161.15,"reasoningOffset":-5.31,"rankEligible":false},{"stateId":"mistral-large-4::enabled","configSlug":"mistral-large-4-thinking","modelId":"mistral-large-4","modelSlug":"mistral-large-4","name":"Mistral Large 4 Preview (thinking)","modelName":"Mistral Large 4 Preview","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":14,"score":159.1,"scoreLo":155.67,"scoreHi":162.95,"reasoningOffset":0.06,"rankEligible":true,"rankMean":58.4453,"rankLo":45.0,"rankHi":70.0},{"stateId":"mistral-large-instruct-2407::default","configSlug":"mistral-large-instruct-2407","modelId":"mistral-large-instruct-2407","modelSlug":"mistral-large-instruct-2407","name":"Mistral Large 2 (Instruct 2407)","modelName":"Mistral Large 2 (Instruct 2407)","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":43,"score":123.85,"scoreLo":120.96,"scoreHi":126.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":450.2039,"rankLo":425.0,"rankHi":476.0},{"stateId":"mistral-large-instruct-2411::default","configSlug":"mistral-large-instruct-2411","modelId":"mistral-large-instruct-2411","modelSlug":"mistral-large-instruct-2411","name":"Mistral Large 2.1 (Instruct 2411)","modelName":"Mistral Large 2.1 (Instruct 2411)","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":32,"score":126.1,"scoreLo":123.1,"scoreHi":128.8,"reasoningOffset":0.0,"rankEligible":true,"rankMean":422.115,"rankLo":397.0,"rankHi":450.0},{"stateId":"mistral-medium-2312::default","configSlug":"mistral-medium","modelId":"mistral-medium-2312","modelSlug":"mistral-medium-2312","name":"Mistral Medium 1.0","modelName":"Mistral Medium 1.0","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":111.14,"scoreLo":105.42,"scoreHi":116.36,"reasoningOffset":0.0,"rankEligible":true,"rankMean":609.3831,"rankLo":559.0,"rankHi":657.0},{"stateId":"mistral-medium-2505::default","configSlug":"mistral-medium-2505","modelId":"mistral-medium-2505","modelSlug":"mistral-medium-2505","name":"Mistral Medium 3","modelName":"Mistral Medium 3","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":38,"score":129.18,"scoreLo":126.48,"scoreHi":131.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":385.8858,"rankLo":358.0,"rankHi":411.0},{"stateId":"mistral-medium-2508::default","configSlug":"mistral-medium-2508","modelId":"mistral-medium-2508","modelSlug":"mistral-medium-2508","name":"Mistral Medium 3.1","modelName":"Mistral Medium 3.1","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":36,"score":135.03,"scoreLo":132.85,"scoreHi":137.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":307.6497,"rankLo":281.0,"rankHi":335.0},{"stateId":"mistral-medium-3-5-128b::default","configSlug":"mistral-medium-3-5-128b","modelId":"mistral-medium-3-5-128b","modelSlug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5","modelName":"Mistral Medium 3.5","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":143.43,"scoreLo":141.13,"scoreHi":145.85,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mistral-medium-3-5-128b::effort:high","configSlug":"mistral-medium-3-5-128b-high","modelId":"mistral-medium-3-5-128b","modelSlug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 (high)","modelName":"Mistral Medium 3.5","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":50,"score":144.32,"scoreLo":142.22,"scoreHi":146.63,"reasoningOffset":0.88,"rankEligible":true,"rankMean":193.37,"rankLo":179.0,"rankHi":208.0},{"stateId":"mistral-medium-3-5-128b::effort:low","configSlug":"mistral-medium-3-5-128b-low","modelId":"mistral-medium-3-5-128b","modelSlug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 (low)","modelName":"Mistral Medium 3.5","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":142.71,"scoreLo":138.89,"scoreHi":146.79,"reasoningOffset":-0.72,"rankEligible":false},{"stateId":"mistral-medium-3-5-128b::effort:medium","configSlug":"mistral-medium-3-5-128b-medium","modelId":"mistral-medium-3-5-128b","modelSlug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 (medium)","modelName":"Mistral Medium 3.5","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":144.17,"scoreLo":140.48,"scoreHi":147.91,"reasoningOffset":0.73,"rankEligible":false},{"stateId":"mistral-medium-3-5-128b::disabled","configSlug":"mistral-medium-3-5-128b-no-thinking","modelId":"mistral-medium-3-5-128b","modelSlug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 (disabled)","modelName":"Mistral Medium 3.5","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":2,"score":136.57,"scoreLo":131.59,"scoreHi":141.49,"reasoningOffset":-6.86,"rankEligible":false},{"stateId":"mistral-nemo-base-2407::default","configSlug":"mistral-nemo-base-2407","modelId":"mistral-nemo-base-2407","modelSlug":"mistral-nemo-base-2407","name":"Mistral NeMo Base 2407","modelName":"Mistral NeMo Base 2407","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":101.54,"scoreLo":94.93,"scoreHi":107.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":706.1744,"rankLo":673.0,"rankHi":738.0},{"stateId":"mistral-nemo-instruct-2407::default","configSlug":"mistral-nemo-instruct-2407","modelId":"mistral-nemo-instruct-2407","modelSlug":"mistral-nemo-instruct-2407","name":"Mistral NeMo Instruct 2407","modelName":"Mistral NeMo Instruct 2407","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":109.47,"scoreLo":104.23,"scoreHi":113.9,"reasoningOffset":0.0,"rankEligible":true,"rankMean":629.4025,"rankLo":597.0,"rankHi":661.0},{"stateId":"mistral-nemo-minitron-8b-base::default","configSlug":"mistral-nemo-minitron-8b-base","modelId":"mistral-nemo-minitron-8b-base","modelSlug":"mistral-nemo-minitron-8b-base","name":"Mistral-NeMo-Minitron 8B Base","modelName":"Mistral-NeMo-Minitron 8B Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":101.06,"scoreLo":92.83,"scoreHi":108.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":709.6014,"rankLo":658.0,"rankHi":757.0},{"stateId":"mistral-nemo-minitron-8b-instruct::default","configSlug":"mistral-nemo-minitron-8b-instruct","modelId":"mistral-nemo-minitron-8b-instruct","modelSlug":"mistral-nemo-minitron-8b-instruct","name":"Mistral-NeMo-Minitron 8B Instruct","modelName":"Mistral-NeMo-Minitron 8B Instruct","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":111.89,"scoreLo":104.67,"scoreHi":118.44,"reasoningOffset":0.0,"rankEligible":true,"rankMean":599.9853,"rankLo":528.975,"rankHi":665.0},{"stateId":"mistral-small-2402::default","configSlug":"mistral-small-2402","modelId":"mistral-small-2402","modelSlug":"mistral-small-2402","name":"Mistral Small 1.0","modelName":"Mistral Small 1.0","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":110.48,"scoreLo":105.22,"scoreHi":115.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":617.4422,"rankLo":577.0,"rankHi":655.0},{"stateId":"mistral-small-24b-base-2501::default","configSlug":"mistral-small-24b-base-2501","modelId":"mistral-small-24b-base-2501","modelSlug":"mistral-small-24b-base-2501","name":"Mistral Small 3 24B Base 2501","modelName":"Mistral Small 3 24B Base 2501","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":116.87,"scoreLo":111.9,"scoreHi":121.33,"reasoningOffset":0.0,"rankEligible":true,"rankMean":540.13,"rankLo":493.975,"rankHi":586.0},{"stateId":"mistral-small-24b-instruct-2501::default","configSlug":"mistral-small-24b-instruct-2501","modelId":"mistral-small-24b-instruct-2501","modelSlug":"mistral-small-24b-instruct-2501","name":"Mistral Small 3 24B Instruct 2501","modelName":"Mistral Small 3 24B Instruct 2501","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":121.15,"scoreLo":117.17,"scoreHi":124.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":487.2133,"rankLo":446.0,"rankHi":525.0},{"stateId":"mistral-small-3-1-24b-base-2503::default","configSlug":"mistral-small-3-1-24b-base-2503","modelId":"mistral-small-3-1-24b-base-2503","modelSlug":"mistral-small-3-1-24b-base-2503","name":"Mistral Small 3.1 24B Base 2503","modelName":"Mistral Small 3.1 24B Base 2503","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":116.48,"scoreLo":111.59,"scoreHi":120.92,"reasoningOffset":0.0,"rankEligible":true,"rankMean":544.6769,"rankLo":501.975,"rankHi":588.0},{"stateId":"mistral-small-3-1-24b-instruct-2503::default","configSlug":"mistral-small-3-1-24b-instruct-2503","modelId":"mistral-small-3-1-24b-instruct-2503","modelSlug":"mistral-small-3-1-24b-instruct-2503","name":"Mistral Small 3.1 24B Instruct 2503","modelName":"Mistral Small 3.1 24B Instruct 2503","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":48,"score":125.1,"scoreLo":122.08,"scoreHi":127.78,"reasoningOffset":0.0,"rankEligible":true,"rankMean":434.265,"rankLo":409.0,"rankHi":461.025},{"stateId":"mistral-small-3-2-24b-instruct-2506::default","configSlug":"mistral-small-3-2-24b-instruct-2506","modelId":"mistral-small-3-2-24b-instruct-2506","modelSlug":"mistral-small-3-2-24b-instruct-2506","name":"Mistral Small 3.2 24B Instruct 2506","modelName":"Mistral Small 3.2 24B Instruct 2506","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":45,"score":126.98,"scoreLo":124.22,"scoreHi":129.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":411.9406,"rankLo":389.0,"rankHi":435.0},{"stateId":"mistral-small-4-119b-2603::default","configSlug":"mistral-small-4-119b-2603","modelId":"mistral-small-4-119b-2603","modelSlug":"mistral-small-4-119b-2603","name":"Mistral Small 4","modelName":"Mistral Small 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":134.35,"scoreLo":131.65,"scoreHi":136.99,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"mistral-small-4-119b-2603::effort:high","configSlug":"mistral-small-4-119b-2603-high","modelId":"mistral-small-4-119b-2603","modelSlug":"mistral-small-4-119b-2603","name":"Mistral Small 4 (high)","modelName":"Mistral Small 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":28,"score":136.21,"scoreLo":134.03,"scoreHi":138.44,"reasoningOffset":1.86,"rankEligible":true,"rankMean":291.5194,"rankLo":266.0,"rankHi":317.0},{"stateId":"mistral-small-4-119b-2603::effort:minimal","configSlug":"mistral-small-4-119b-2603-minimal","modelId":"mistral-small-4-119b-2603","modelSlug":"mistral-small-4-119b-2603","name":"Mistral Small 4 (minimal)","modelName":"Mistral Small 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":129.82,"scoreLo":124.74,"scoreHi":134.76,"reasoningOffset":-4.53,"rankEligible":false},{"stateId":"mistral-small-4-119b-2603::disabled","configSlug":"mistral-small-4-119b-2603-no-thinking","modelId":"mistral-small-4-119b-2603","modelSlug":"mistral-small-4-119b-2603","name":"Mistral Small 4 (disabled)","modelName":"Mistral Small 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":9,"score":131.72,"scoreLo":128.94,"scoreHi":134.25,"reasoningOffset":-2.63,"rankEligible":true},{"stateId":"mistral-small-4-119b-2603::enabled","configSlug":"mistral-small-4-119b-2603-thinking-context-arena-mrcr-v2","modelId":"mistral-small-4-119b-2603","modelSlug":"mistral-small-4-119b-2603","name":"Mistral Small 4 (thinking)","modelName":"Mistral Small 4","organizationSlug":"mistral-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":135.49,"scoreLo":131.23,"scoreHi":139.72,"reasoningOffset":1.14,"rankEligible":false},{"stateId":"mistral-small-instruct-2409::default","configSlug":"mistral-small-instruct-2409","modelId":"mistral-small-instruct-2409","modelSlug":"mistral-small-instruct-2409","name":"Mistral Small Instruct 2409","modelName":"Mistral Small Instruct 2409","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":110.73,"scoreLo":105.54,"scoreHi":115.16,"reasoningOffset":0.0,"rankEligible":true,"rankMean":614.5856,"rankLo":576.975,"rankHi":650.0},{"stateId":"mixtral-8x22b-instruct-v0-1::default","configSlug":"mixtral-8x22b-instruct-v0-1","modelId":"mixtral-8x22b-instruct-v0-1","modelSlug":"mixtral-8x22b-instruct-v0-1","name":"Mixtral 8x22B Instruct","modelName":"Mixtral 8x22B Instruct","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":60,"score":115.19,"scoreLo":111.51,"scoreHi":118.44,"reasoningOffset":0.0,"rankEligible":true,"rankMean":559.3053,"rankLo":540.0,"rankHi":580.0},{"stateId":"mixtral-8x22b-v0-1::default","configSlug":"mixtral-8x22b-v0-1","modelId":"mixtral-8x22b-v0-1","modelSlug":"mixtral-8x22b-v0-1","name":"Mixtral 8x22B","modelName":"Mixtral 8x22B","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":47,"score":113.58,"scoreLo":109.48,"scoreHi":117.11,"reasoningOffset":0.0,"rankEligible":true,"rankMean":579.3144,"rankLo":555.0,"rankHi":603.0},{"stateId":"mixtral-8x7b-instruct-v0-1::default","configSlug":"mixtral-8x7b-instruct-v0-1","modelId":"mixtral-8x7b-instruct-v0-1","modelSlug":"mixtral-8x7b-instruct-v0-1","name":"Mixtral 8x7B Instruct","modelName":"Mixtral 8x7B Instruct","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":37,"score":105.5,"scoreLo":100.2,"scoreHi":109.92,"reasoningOffset":0.0,"rankEligible":true,"rankMean":671.9961,"rankLo":650.0,"rankHi":692.0},{"stateId":"mixtral-8x7b-v0-1::default","configSlug":"mixtral-8x7b-v0-1","modelId":"mixtral-8x7b-v0-1","modelSlug":"mixtral-8x7b-v0-1","name":"Mixtral 8x7B","modelName":"Mixtral 8x7B","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":105.56,"scoreLo":100.11,"scoreHi":110.22,"reasoningOffset":0.0,"rankEligible":true,"rankMean":671.2286,"rankLo":646.0,"rankHi":694.0},{"stateId":"molmo-7b-d-0924::default","configSlug":"molmo-7b-d-0924","modelId":"molmo-7b-d-0924","modelSlug":"molmo-7b-d-0924","name":"Molmo 7B-D","modelName":"Molmo 7B-D","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":101.72,"scoreLo":94.99,"scoreHi":107.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":704.6778,"rankLo":672.0,"rankHi":737.0},{"stateId":"molmo2-8b::default","configSlug":"molmo2-8b","modelId":"molmo2-8b","modelSlug":"molmo2-8b","name":"Molmo 2 8B","modelName":"Molmo 2 8B","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":111.32,"scoreLo":100.99,"scoreHi":120.34,"reasoningOffset":0.0,"rankEligible":true,"rankMean":605.3792,"rankLo":506.0,"rankHi":705.025},{"stateId":"moonlight-16b-a3b::default","configSlug":"moonlight-16b-a3b","modelId":"moonlight-16b-a3b","modelSlug":"moonlight-16b-a3b","name":"Moonlight 16B A3B","modelName":"Moonlight 16B A3B","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":108.71,"scoreLo":103.03,"scoreHi":113.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":637.9672,"rankLo":601.0,"rankHi":673.0},{"stateId":"moonlight-16b-a3b-instruct::default","configSlug":"moonlight-16b-a3b-instruct","modelId":"moonlight-16b-a3b-instruct","modelSlug":"moonlight-16b-a3b-instruct","name":"Moonlight 16B A3B Instruct","modelName":"Moonlight 16B A3B Instruct","organizationSlug":"moonshot-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":106.12,"scoreLo":98.94,"scoreHi":112.09,"reasoningOffset":0.0,"rankEligible":true,"rankMean":664.755,"rankLo":614.0,"rankHi":710.0},{"stateId":"motif-2-12-7b-reasoning::default","configSlug":"motif-2-12-7b-reasoning","modelId":"motif-2-12-7b-reasoning","modelSlug":"motif-2-12-7b-reasoning","name":"Motif-2-12.7B-Reasoning","modelName":"Motif-2-12.7B-Reasoning","organizationSlug":"motif","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":131.2,"scoreLo":126.4,"scoreHi":135.81,"reasoningOffset":2.4,"rankEligible":true,"rankMean":358.7433,"rankLo":300.0,"rankHi":413.025},{"stateId":"motif-3::default","configSlug":"motif-3","modelId":"motif-3","modelSlug":"motif-3","name":"Motif 3","modelName":"Motif 3","organizationSlug":"motif","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":153.24,"scoreLo":149.66,"scoreHi":157.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":100.0689,"rankLo":80.0,"rankHi":127.0},{"stateId":"motif-3-beta::default","configSlug":"motif-3-beta","modelId":"motif-3-beta","modelSlug":"motif-3-beta","name":"Motif-3-Beta","modelName":"Motif-3-Beta","organizationSlug":"motif","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":154.54,"scoreLo":150.61,"scoreHi":158.78,"reasoningOffset":0.0,"rankEligible":true,"rankMean":89.6811,"rankLo":72.975,"rankHi":115.0},{"stateId":"mpt-30b::default","configSlug":"mpt-30b","modelId":"mpt-30b","modelSlug":"mpt-30b","name":"MPT-30B","modelName":"MPT-30B","organizationSlug":"mosaicml","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":86.18,"scoreLo":77.74,"scoreHi":93.27,"reasoningOffset":0.0,"rankEligible":true,"rankMean":809.0614,"rankLo":793.0,"rankHi":825.0},{"stateId":"mpt-7b::default","configSlug":"mpt-7b","modelId":"mpt-7b","modelSlug":"mpt-7b","name":"MPT-7B","modelName":"MPT-7B","organizationSlug":"mosaicml","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":74.42,"scoreLo":63.54,"scoreHi":83.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":850.3564,"rankLo":838.0,"rankHi":861.0},{"stateId":"muse-glimmer-30b::default","configSlug":"muse-glimmer-30b","modelId":"muse-glimmer-30b","modelSlug":"muse-glimmer-30b","name":"Muse Glimmer 30B","modelName":"Muse Glimmer 30B","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":149.31,"scoreLo":146.85,"scoreHi":152.27,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"muse-glimmer-30b::effort:high","configSlug":"muse-glimmer-30b-high","modelId":"muse-glimmer-30b","modelSlug":"muse-glimmer-30b","name":"Muse Glimmer 30B (high)","modelName":"Muse Glimmer 30B","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":31,"score":150.02,"scoreLo":147.54,"scoreHi":152.77,"reasoningOffset":0.71,"rankEligible":true},{"stateId":"muse-glimmer-30b::effort:low","configSlug":"muse-glimmer-30b-low","modelId":"muse-glimmer-30b","modelSlug":"muse-glimmer-30b","name":"Muse Glimmer 30B (low)","modelName":"Muse Glimmer 30B","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":11,"score":146.17,"scoreLo":143.45,"scoreHi":149.24,"reasoningOffset":-3.13,"rankEligible":true},{"stateId":"muse-glimmer-30b::effort:medium","configSlug":"muse-glimmer-30b-medium","modelId":"muse-glimmer-30b","modelSlug":"muse-glimmer-30b","name":"Muse Glimmer 30B (medium)","modelName":"Muse Glimmer 30B","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":11,"score":148.76,"scoreLo":146.02,"scoreHi":151.95,"reasoningOffset":-0.55,"rankEligible":true},{"stateId":"muse-glimmer-30b::effort:xhigh","configSlug":"muse-glimmer-30b-xhigh","modelId":"muse-glimmer-30b","modelSlug":"muse-glimmer-30b","name":"Muse Glimmer 30B (xhigh)","modelName":"Muse Glimmer 30B","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":25,"score":150.39,"scoreLo":147.74,"scoreHi":153.42,"reasoningOffset":1.09,"rankEligible":true,"rankMean":127.3972,"rankLo":112.0,"rankHi":144.0},{"stateId":"muse-spark::default","configSlug":"muse-spark","modelId":"muse-spark","modelSlug":"muse-spark","name":"Muse Spark","modelName":"Muse Spark","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":153.4,"scoreLo":150.68,"scoreHi":156.51,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"muse-spark-1-1::default","configSlug":"muse-spark-1-1","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"default","reasoningCondition":"default","isDefault":true,"n":36,"score":158.5,"scoreLo":155.44,"scoreHi":162.33,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"muse-spark-1-1::effort:high","configSlug":"muse-spark-1-1-high","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1 (high)","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":20,"score":157.96,"scoreLo":154.9,"scoreHi":161.64,"reasoningOffset":-0.54,"rankEligible":true},{"stateId":"muse-spark-1-1::effort:low","configSlug":"muse-spark-1-1-low","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1 (low)","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":7,"score":155.24,"scoreLo":151.92,"scoreHi":159.03,"reasoningOffset":-3.25,"rankEligible":true},{"stateId":"muse-spark-1-1::effort:max","configSlug":"muse-spark-1-1-max-reactbench-v1-meta-agent","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1 (max)","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":2,"score":158.83,"scoreLo":155.27,"scoreHi":163.07,"reasoningOffset":0.34,"rankEligible":false},{"stateId":"muse-spark-1-1::effort:medium","configSlug":"muse-spark-1-1-medium","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1 (medium)","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":10,"score":157.08,"scoreLo":153.75,"scoreHi":161.18,"reasoningOffset":-1.42,"rankEligible":true},{"stateId":"muse-spark-1-1::effort:minimal","configSlug":"muse-spark-1-1-minimal","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1 (minimal)","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":4,"score":156.3,"scoreLo":152.33,"scoreHi":160.94,"reasoningOffset":-2.2,"rankEligible":true},{"stateId":"muse-spark-1-1::enabled","configSlug":"muse-spark-1-1-thinking","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1 (thinking)","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":3,"score":159.77,"scoreLo":156.05,"scoreHi":164.15,"reasoningOffset":1.28,"rankEligible":false},{"stateId":"muse-spark-1-1::effort:xhigh","configSlug":"muse-spark-1-1-xhigh","modelId":"muse-spark-1-1","modelSlug":"muse-spark-1-1","name":"Muse Spark 1.1 (xhigh)","modelName":"Muse Spark 1.1","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":72,"score":159.1,"scoreLo":156.0,"scoreHi":162.88,"reasoningOffset":0.6,"rankEligible":true,"rankMean":58.6331,"rankLo":50.0,"rankHi":66.0},{"stateId":"muse-spark-1-2::default","configSlug":"muse-spark-1-2","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":158.7,"scoreLo":155.56,"scoreHi":162.48,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"muse-spark-1-2::effort:auto","configSlug":"muse-spark-1-2-auto","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2 (auto)","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"auto","reasoningCondition":"effort:auto","isDefault":false,"n":1,"score":158.0,"scoreLo":152.75,"scoreHi":163.64,"reasoningOffset":-0.7,"rankEligible":false},{"stateId":"muse-spark-1-2::effort:high","configSlug":"muse-spark-1-2-high","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2 (high)","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":11,"score":159.13,"scoreLo":155.83,"scoreHi":163.1,"reasoningOffset":0.43,"rankEligible":true,"rankMean":57.9922,"rankLo":42.0,"rankHi":71.0},{"stateId":"muse-spark-1-2::enabled","configSlug":"muse-spark-1-2-jobbench-opencode","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2 (enabled)","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":159.0,"scoreLo":155.09,"scoreHi":163.53,"reasoningOffset":0.3,"rankEligible":false},{"stateId":"muse-spark-1-2::effort:low","configSlug":"muse-spark-1-2-low","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2 (low)","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":4,"score":156.9,"scoreLo":153.09,"scoreHi":161.37,"reasoningOffset":-1.8,"rankEligible":true},{"stateId":"muse-spark-1-2::effort:medium","configSlug":"muse-spark-1-2-medium","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2 (medium)","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":5,"score":158.29,"scoreLo":154.39,"scoreHi":162.58,"reasoningOffset":-0.41,"rankEligible":true},{"stateId":"muse-spark-1-2::effort:minimal","configSlug":"muse-spark-1-2-minimal","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2 (minimal)","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":4,"score":155.0,"scoreLo":151.27,"scoreHi":159.52,"reasoningOffset":-3.7,"rankEligible":true},{"stateId":"muse-spark-1-2::effort:xhigh","configSlug":"muse-spark-1-2-xhigh","modelId":"muse-spark-1-2","modelSlug":"muse-spark-1-2","name":"Muse Spark 1.2 (xhigh)","modelName":"Muse Spark 1.2","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":82,"score":158.66,"scoreLo":155.62,"scoreHi":162.41,"reasoningOffset":-0.04,"rankEligible":true},{"stateId":"muse-spark-1-3::default","configSlug":"muse-spark-1-3","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":30,"score":160.79,"scoreLo":157.52,"scoreHi":164.73,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"muse-spark-1-3::effort:high","configSlug":"muse-spark-1-3-high","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3 (high)","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":22,"score":160.06,"scoreLo":156.81,"scoreHi":164.05,"reasoningOffset":-0.73,"rankEligible":true},{"stateId":"muse-spark-1-3::enabled","configSlug":"muse-spark-1-3-jobbench-opencode","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3 (enabled)","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":161.97,"scoreLo":157.98,"scoreHi":166.8,"reasoningOffset":1.18,"rankEligible":false},{"stateId":"muse-spark-1-3::effort:low","configSlug":"muse-spark-1-3-low","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3 (low)","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":7,"score":158.2,"scoreLo":154.82,"scoreHi":162.23,"reasoningOffset":-2.59,"rankEligible":true},{"stateId":"muse-spark-1-3::effort:max","configSlug":"muse-spark-1-3-max","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3 (max)","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":83,"score":162.13,"scoreLo":158.71,"scoreHi":166.28,"reasoningOffset":1.34,"rankEligible":true,"rankMean":26.5578,"rankLo":22.0,"rankHi":32.0},{"stateId":"muse-spark-1-3::effort:medium","configSlug":"muse-spark-1-3-medium","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3 (medium)","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":8,"score":159.91,"scoreLo":156.46,"scoreHi":164.02,"reasoningOffset":-0.89,"rankEligible":true},{"stateId":"muse-spark-1-3::effort:minimal","configSlug":"muse-spark-1-3-minimal","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3 (minimal)","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":4,"score":155.48,"scoreLo":152.12,"scoreHi":159.4,"reasoningOffset":-5.32,"rankEligible":true},{"stateId":"muse-spark-1-3::effort:xhigh","configSlug":"muse-spark-1-3-xhigh","modelId":"muse-spark-1-3","modelSlug":"muse-spark-1-3","name":"Muse Spark 1.3 (xhigh)","modelName":"Muse Spark 1.3","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":63,"score":160.68,"scoreLo":157.42,"scoreHi":164.61,"reasoningOffset":-0.12,"rankEligible":true},{"stateId":"muse-spark-open::default","configSlug":"muse-spark-open","modelId":"muse-spark-open","modelSlug":"muse-spark-open","name":"Muse Spark Open","modelName":"Muse Spark Open","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":158.27,"scoreLo":155.03,"scoreHi":162.17,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"muse-spark-open::effort:high","configSlug":"muse-spark-open-high","modelId":"muse-spark-open","modelSlug":"muse-spark-open","name":"Muse Spark Open (high)","modelName":"Muse Spark Open","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":10,"score":157.55,"scoreLo":154.47,"scoreHi":161.27,"reasoningOffset":-0.72,"rankEligible":true},{"stateId":"muse-spark-open::effort:low","configSlug":"muse-spark-open-low","modelId":"muse-spark-open","modelSlug":"muse-spark-open","name":"Muse Spark Open (low)","modelName":"Muse Spark Open","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":154.31,"scoreLo":151.48,"scoreHi":157.87,"reasoningOffset":-3.96,"rankEligible":false},{"stateId":"muse-spark-open::effort:medium","configSlug":"muse-spark-open-medium","modelId":"muse-spark-open","modelSlug":"muse-spark-open","name":"Muse Spark Open (medium)","modelName":"Muse Spark Open","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":3,"score":156.28,"scoreLo":153.16,"scoreHi":160.0,"reasoningOffset":-1.99,"rankEligible":false},{"stateId":"muse-spark-open::effort:xhigh","configSlug":"muse-spark-open-xhigh","modelId":"muse-spark-open","modelSlug":"muse-spark-open","name":"Muse Spark Open (xhigh)","modelName":"Muse Spark Open","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":49,"score":158.45,"scoreLo":155.34,"scoreHi":162.08,"reasoningOffset":0.18,"rankEligible":true,"rankMean":65.0081,"rankLo":56.0,"rankHi":71.0},{"stateId":"muse-spark::enabled","configSlug":"muse-spark-thinking","modelId":"muse-spark","modelSlug":"muse-spark","name":"Muse Spark (thinking)","modelName":"Muse Spark","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":31,"score":154.69,"scoreLo":151.9,"scoreHi":157.9,"reasoningOffset":1.29,"rankEligible":true,"rankMean":87.1844,"rankLo":80.0,"rankHi":97.0},{"stateId":"muse-wonder::default","configSlug":"muse-wonder","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder","modelName":"Muse Wonder","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":27,"score":163.96,"scoreLo":160.24,"scoreHi":168.41,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"muse-wonder-full::effort:max","configSlug":"muse-wonder-full-max","modelId":"muse-wonder-full","modelSlug":"muse-wonder-full","name":"Muse Wonder Full (max)","modelName":"Muse Wonder Full","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":45,"score":165.73,"scoreLo":161.91,"scoreHi":170.28,"reasoningOffset":0.86,"rankEligible":true,"rankMean":10.9239,"rankLo":9.0,"rankHi":14.0},{"stateId":"muse-wonder::effort:high","configSlug":"muse-wonder-high","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder (high)","modelName":"Muse Wonder","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":16,"score":163.52,"scoreLo":159.89,"scoreHi":167.86,"reasoningOffset":-0.43,"rankEligible":true},{"stateId":"muse-wonder::effort:low","configSlug":"muse-wonder-low","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder (low)","modelName":"Muse Wonder","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":6,"score":161.19,"scoreLo":157.76,"scoreHi":165.26,"reasoningOffset":-2.76,"rankEligible":true},{"stateId":"muse-wonder::effort:max","configSlug":"muse-wonder-max","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder (max)","modelName":"Muse Wonder","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":77,"score":164.54,"scoreLo":160.81,"scoreHi":168.94,"reasoningOffset":0.59,"rankEligible":true,"rankMean":15.9939,"rankLo":14.0,"rankHi":18.0},{"stateId":"muse-wonder::effort:medium","configSlug":"muse-wonder-medium","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder (medium)","modelName":"Muse Wonder","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":6,"score":162.89,"scoreLo":159.3,"scoreHi":167.08,"reasoningOffset":-1.07,"rankEligible":true},{"stateId":"muse-wonder::effort:minimal","configSlug":"muse-wonder-minimal","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder (minimal)","modelName":"Muse Wonder","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":3,"score":158.59,"scoreLo":154.74,"scoreHi":162.8,"reasoningOffset":-5.36,"rankEligible":false},{"stateId":"muse-wonder::effort:xhigh","configSlug":"muse-wonder-xhigh","modelId":"muse-wonder","modelSlug":"muse-wonder","name":"Muse Wonder (xhigh)","modelName":"Muse Wonder","organizationSlug":"meta","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":5,"score":163.93,"scoreLo":160.22,"scoreHi":168.47,"reasoningOffset":-0.02,"rankEligible":true},{"stateId":"mythomax-l2-13b::default","configSlug":"mythomax-l2-13b","modelId":"mythomax-l2-13b","modelSlug":"mythomax-l2-13b","name":"MythoMax L2 13B","modelName":"MythoMax L2 13B","organizationSlug":"gryphe","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":95.64,"scoreLo":86.85,"scoreHi":103.26,"reasoningOffset":0.0,"rankEligible":true,"rankMean":752.5297,"rankLo":709.0,"rankHi":790.0},{"stateId":"nanbeige4-1-3b::default","configSlug":"nanbeige4-1-3b","modelId":"nanbeige4-1-3b","modelSlug":"nanbeige4-1-3b","name":"Nanbeige4.1-3B","modelName":"Nanbeige4.1-3B","organizationSlug":"nanbeige","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":111.58,"scoreLo":97.64,"scoreHi":122.96,"reasoningOffset":2.4,"rankEligible":true,"rankMean":600.545,"rankLo":468.0,"rankHi":730.0},{"stateId":"nanbeige4-3-33b::enabled","configSlug":"nanbeige4-3-33b-thinking","modelId":"nanbeige4-3-33b","modelSlug":"nanbeige4-3-33b","name":"Nanbeige4.3-33B (thinking)","modelName":"Nanbeige4.3-33B","organizationSlug":"nanbeige","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":17,"score":158.7,"scoreLo":155.42,"scoreHi":162.59,"reasoningOffset":1.07,"rankEligible":true,"rankMean":62.3561,"rankLo":49.0,"rankHi":73.0},{"stateId":"nanbeige4-3b-thinking-2511::default","configSlug":"nanbeige4-3b-thinking-2511","modelId":"nanbeige4-3b-thinking-2511","modelSlug":"nanbeige4-3b-thinking-2511","name":"Nanbeige4-3B-Thinking-2511","modelName":"Nanbeige4-3B-Thinking-2511","organizationSlug":"nanbeige","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":4,"score":122.24,"scoreLo":113.36,"scoreHi":130.51,"reasoningOffset":2.4,"rankEligible":true,"rankMean":472.3047,"rankLo":372.975,"rankHi":572.0},{"stateId":"nemotron-3-5-lightning-30b-a3b-base-bf16::default","configSlug":"nemotron-3-5-lightning-30b-a3b-base-bf16","modelId":"nemotron-3-5-lightning-30b-a3b-base-bf16","modelSlug":"nemotron-3-5-lightning-30b-a3b-base-bf16","name":"Nemotron 3.5 Lightning 30B A3B Base","modelName":"Nemotron 3.5 Lightning 30B A3B Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":119.92,"scoreLo":115.52,"scoreHi":123.82,"reasoningOffset":0.0,"rankEligible":true,"rankMean":503.3431,"rankLo":460.975,"rankHi":543.0},{"stateId":"nemotron-3-5-lightning-30b-a3b-bf16::default","configSlug":"nemotron-3-5-lightning-30b-a3b-bf16","modelId":"nemotron-3-5-lightning-30b-a3b-bf16","modelSlug":"nemotron-3-5-lightning-30b-a3b-bf16","name":"Nemotron 3.5 Lightning 30B A3B BF16","modelName":"Nemotron 3.5 Lightning 30B A3B BF16","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":137.91,"scoreLo":135.32,"scoreHi":140.45,"reasoningOffset":0.0,"rankEligible":true,"rankMean":268.3986,"rankLo":239.0,"rankHi":299.0},{"stateId":"nemotron-3-5-lightning-30b-a3b-bf16::enabled","configSlug":"nemotron-3-5-lightning-30b-a3b-bf16-thinking","modelId":"nemotron-3-5-lightning-30b-a3b-bf16","modelSlug":"nemotron-3-5-lightning-30b-a3b-bf16","name":"Nemotron 3.5 Lightning 30B A3B BF16 (thinking)","modelName":"Nemotron 3.5 Lightning 30B A3B BF16","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":139.34,"scoreLo":135.54,"scoreHi":142.94,"reasoningOffset":1.44,"rankEligible":false},{"stateId":"nemotron-3-5-lightning-30b-a3b-nvfp4::default","configSlug":"nemotron-3-5-lightning-30b-a3b-nvfp4","modelId":"nemotron-3-5-lightning-30b-a3b-nvfp4","modelSlug":"nemotron-3-5-lightning-30b-a3b-nvfp4","name":"Nemotron 3.5 Lightning 30B A3B NVFP4","modelName":"Nemotron 3.5 Lightning 30B A3B NVFP4","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":139.4,"scoreLo":136.31,"scoreHi":142.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":248.2764,"rankLo":214.0,"rankHi":287.0},{"stateId":"nemotron-3-5-super-vl-120b-a12b-bf16::default","configSlug":"nemotron-3-5-super-vl-120b-a12b-bf16","modelId":"nemotron-3-5-super-vl-120b-a12b-bf16","modelSlug":"nemotron-3-5-super-vl-120b-a12b-bf16","name":"Nemotron 3.5 Super VL 120B A12B BF16","modelName":"Nemotron 3.5 Super VL 120B A12B BF16","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":153.54,"scoreLo":150.32,"scoreHi":157.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":97.2319,"rankLo":81.0,"rankHi":115.0},{"stateId":"nemotron-3-5-super-vl-120b-a12b-bf16::enabled","configSlug":"nemotron-3-5-super-vl-120b-a12b-bf16-thinking","modelId":"nemotron-3-5-super-vl-120b-a12b-bf16","modelSlug":"nemotron-3-5-super-vl-120b-a12b-bf16","name":"Nemotron 3.5 Super VL 120B A12B BF16 (thinking)","modelName":"Nemotron 3.5 Super VL 120B A12B BF16","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":154.4,"scoreLo":150.41,"scoreHi":159.04,"reasoningOffset":0.86,"rankEligible":false},{"stateId":"nemotron-3-5-super-vl-120b-a12b-nvfp4::default","configSlug":"nemotron-3-5-super-vl-120b-a12b-nvfp4","modelId":"nemotron-3-5-super-vl-120b-a12b-nvfp4","modelSlug":"nemotron-3-5-super-vl-120b-a12b-nvfp4","name":"Nemotron 3.5 Super VL 120B A12B NVFP4","modelName":"Nemotron 3.5 Super VL 120B A12B NVFP4","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":153.39,"scoreLo":150.27,"scoreHi":157.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":98.5681,"rankLo":82.0,"rankHi":116.0},{"stateId":"nemotron-3-5-super-vl-120b-a12b-nvfp4::enabled","configSlug":"nemotron-3-5-super-vl-120b-a12b-nvfp4-thinking","modelId":"nemotron-3-5-super-vl-120b-a12b-nvfp4","modelSlug":"nemotron-3-5-super-vl-120b-a12b-nvfp4","name":"Nemotron 3.5 Super VL 120B A12B NVFP4 (thinking)","modelName":"Nemotron 3.5 Super VL 120B A12B NVFP4","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":153.93,"scoreLo":149.83,"scoreHi":158.38,"reasoningOffset":0.54,"rankEligible":false},{"stateId":"nemotron-3-nano-30b-a3b-base-bf16::default","configSlug":"nemotron-3-nano-30b-a3b-base-bf16","modelId":"nemotron-3-nano-30b-a3b-base-bf16","modelSlug":"nemotron-3-nano-30b-a3b-base-bf16","name":"Nemotron 3 Nano 30B A3B Base","modelName":"Nemotron 3 Nano 30B A3B Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":120.87,"scoreLo":116.23,"scoreHi":125.24,"reasoningOffset":0.0,"rankEligible":true,"rankMean":490.6231,"rankLo":438.0,"rankHi":537.0},{"stateId":"nemotron-3-nano-30b-a3b-bf16::default","configSlug":"nemotron-3-nano-30b-a3b-bf16","modelId":"nemotron-3-nano-30b-a3b-bf16","modelSlug":"nemotron-3-nano-30b-a3b-bf16","name":"Nemotron 3 Nano 30B A3B","modelName":"Nemotron 3 Nano 30B A3B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":136.5,"scoreLo":134.12,"scoreHi":138.9,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nemotron-3-nano-30b-a3b-bf16::effort:high","configSlug":"nemotron-3-nano-30b-a3b-bf16-high","modelId":"nemotron-3-nano-30b-a3b-bf16","modelSlug":"nemotron-3-nano-30b-a3b-bf16","name":"Nemotron 3 Nano 30B A3B (high)","modelName":"Nemotron 3 Nano 30B A3B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":137.15,"scoreLo":133.85,"scoreHi":140.51,"reasoningOffset":0.65,"rankEligible":false},{"stateId":"nemotron-3-nano-30b-a3b-bf16::disabled","configSlug":"nemotron-3-nano-30b-a3b-bf16-no-thinking","modelId":"nemotron-3-nano-30b-a3b-bf16","modelSlug":"nemotron-3-nano-30b-a3b-bf16","name":"Nemotron 3 Nano 30B A3B (disabled)","modelName":"Nemotron 3 Nano 30B A3B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":9,"score":127.51,"scoreLo":123.53,"scoreHi":131.03,"reasoningOffset":-8.99,"rankEligible":true},{"stateId":"nemotron-3-nano-30b-a3b-bf16::enabled","configSlug":"nemotron-3-nano-30b-a3b-bf16-thinking","modelId":"nemotron-3-nano-30b-a3b-bf16","modelSlug":"nemotron-3-nano-30b-a3b-bf16","name":"Nemotron 3 Nano 30B A3B (thinking)","modelName":"Nemotron 3 Nano 30B A3B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":15,"score":137.47,"scoreLo":135.01,"scoreHi":140.01,"reasoningOffset":0.97,"rankEligible":true,"rankMean":274.42,"rankLo":246.0,"rankHi":303.0},{"stateId":"nemotron-3-nano-4b-bf16::default","configSlug":"nemotron-3-nano-4b-bf16","modelId":"nemotron-3-nano-4b-bf16","modelSlug":"nemotron-3-nano-4b-bf16","name":"Nemotron 3 Nano 4B","modelName":"Nemotron 3 Nano 4B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":125.2,"scoreLo":121.26,"scoreHi":128.85,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nemotron-3-nano-4b-bf16::disabled","configSlug":"nemotron-3-nano-4b-bf16-no-thinking","modelId":"nemotron-3-nano-4b-bf16","modelSlug":"nemotron-3-nano-4b-bf16","name":"Nemotron 3 Nano 4B (disabled)","modelName":"Nemotron 3 Nano 4B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":6,"score":122.86,"scoreLo":116.61,"scoreHi":128.29,"reasoningOffset":-2.34,"rankEligible":true},{"stateId":"nemotron-3-nano-4b-bf16::enabled","configSlug":"nemotron-3-nano-4b-bf16-thinking","modelId":"nemotron-3-nano-4b-bf16","modelSlug":"nemotron-3-nano-4b-bf16","name":"Nemotron 3 Nano 4B (thinking)","modelName":"Nemotron 3 Nano 4B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":9,"score":127.86,"scoreLo":124.42,"scoreHi":131.03,"reasoningOffset":2.66,"rankEligible":true,"rankMean":401.6436,"rankLo":366.0,"rankHi":435.0},{"stateId":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16::default","configSlug":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","modelId":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","modelSlug":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","name":"Nemotron 3 Nano Omni 30B A3B","modelName":"Nemotron 3 Nano Omni 30B A3B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":134.01,"scoreLo":130.58,"scoreHi":137.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":321.4936,"rankLo":280.0,"rankHi":363.0},{"stateId":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16::disabled","configSlug":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16-no-thinking","modelId":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","modelSlug":"nemotron-3-nano-omni-30b-a3b-reasoning-bf16","name":"Nemotron 3 Nano Omni 30B A3B (disabled)","modelName":"Nemotron 3 Nano Omni 30B A3B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":3,"score":132.57,"scoreLo":128.81,"scoreHi":136.23,"reasoningOffset":-1.44,"rankEligible":false},{"stateId":"nemotron-3-super-120b-a12b-base-bf16::default","configSlug":"nemotron-3-super-120b-a12b-base-bf16","modelId":"nemotron-3-super-120b-a12b-base-bf16","modelSlug":"nemotron-3-super-120b-a12b-base-bf16","name":"Nemotron 3 Super 120B A12B Base","modelName":"Nemotron 3 Super 120B A12B Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":128.44,"scoreLo":124.09,"scoreHi":132.7,"reasoningOffset":0.0,"rankEligible":true,"rankMean":394.22,"rankLo":343.0,"rankHi":440.0},{"stateId":"nemotron-3-super-120b-a12b-bf16::default","configSlug":"nemotron-3-super-120b-a12b-bf16","modelId":"nemotron-3-super-120b-a12b-bf16","modelSlug":"nemotron-3-super-120b-a12b-bf16","name":"Nemotron 3 Super 120B A12B","modelName":"Nemotron 3 Super 120B A12B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":141.7,"scoreLo":139.52,"scoreHi":144.06,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nemotron-3-super-120b-a12b-bf16::enabled","configSlug":"nemotron-3-super-120b-a12b-bf16-thinking","modelId":"nemotron-3-super-120b-a12b-bf16","modelSlug":"nemotron-3-super-120b-a12b-bf16","name":"Nemotron 3 Super 120B A12B (thinking)","modelName":"Nemotron 3 Super 120B A12B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":23,"score":144.05,"scoreLo":141.91,"scoreHi":146.33,"reasoningOffset":2.35,"rankEligible":true,"rankMean":196.1117,"rankLo":179.0,"rankHi":211.025},{"stateId":"nemotron-3-ultra-550b-a55b-base-bf16::default","configSlug":"nemotron-3-ultra-550b-a55b-base-bf16","modelId":"nemotron-3-ultra-550b-a55b-base-bf16","modelSlug":"nemotron-3-ultra-550b-a55b-base-bf16","name":"Nemotron 3 Ultra 550B A55B Base","modelName":"Nemotron 3 Ultra 550B A55B Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":129.28,"scoreLo":125.51,"scoreHi":133.02,"reasoningOffset":0.0,"rankEligible":true,"rankMean":383.8853,"rankLo":339.0,"rankHi":423.0},{"stateId":"nemotron-3-ultra-550b-a55b-bf16::default","configSlug":"nemotron-3-ultra-550b-a55b-bf16","modelId":"nemotron-3-ultra-550b-a55b-bf16","modelSlug":"nemotron-3-ultra-550b-a55b-bf16","name":"Nemotron 3 Ultra 550B A55B","modelName":"Nemotron 3 Ultra 550B A55B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":26,"score":148.41,"scoreLo":146.06,"scoreHi":151.07,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nemotron-3-ultra-550b-a55b-bf16::effort:high","configSlug":"nemotron-3-ultra-550b-a55b-bf16-high","modelId":"nemotron-3-ultra-550b-a55b-bf16","modelSlug":"nemotron-3-ultra-550b-a55b-bf16","name":"Nemotron 3 Ultra 550B A55B (high)","modelName":"Nemotron 3 Ultra 550B A55B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":148.87,"scoreLo":145.49,"scoreHi":152.68,"reasoningOffset":0.46,"rankEligible":false},{"stateId":"nemotron-3-ultra-550b-a55b-bf16::disabled","configSlug":"nemotron-3-ultra-550b-a55b-bf16-no-thinking","modelId":"nemotron-3-ultra-550b-a55b-bf16","modelSlug":"nemotron-3-ultra-550b-a55b-bf16","name":"Nemotron 3 Ultra 550B A55B (disabled)","modelName":"Nemotron 3 Ultra 550B A55B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":143.2,"scoreLo":136.48,"scoreHi":150.32,"reasoningOffset":-5.21,"rankEligible":false},{"stateId":"nemotron-3-ultra-550b-a55b-bf16::enabled","configSlug":"nemotron-3-ultra-550b-a55b-bf16-thinking","modelId":"nemotron-3-ultra-550b-a55b-bf16","modelSlug":"nemotron-3-ultra-550b-a55b-bf16","name":"Nemotron 3 Ultra 550B A55B (thinking)","modelName":"Nemotron 3 Ultra 550B A55B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":26,"score":148.64,"scoreLo":146.25,"scoreHi":151.34,"reasoningOffset":0.24,"rankEligible":true,"rankMean":146.4444,"rankLo":131.0,"rankHi":164.0},{"stateId":"nemotron-3-ultra-550b-a55b-nvfp4::default","configSlug":"nemotron-3-ultra-550b-a55b-nvfp4","modelId":"nemotron-3-ultra-550b-a55b-nvfp4","modelSlug":"nemotron-3-ultra-550b-a55b-nvfp4","name":"Nemotron 3 Ultra 550B A55B (NVFP4)","modelName":"Nemotron 3 Ultra 550B A55B (NVFP4)","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":34,"score":147.36,"scoreLo":144.98,"scoreHi":150.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":160.7408,"rankLo":144.0,"rankHi":178.0},{"stateId":"nemotron-4-15b::default","configSlug":"nemotron-4-15b","modelId":"nemotron-4-15b","modelSlug":"nemotron-4-15b","name":"Nemotron-4 15B","modelName":"Nemotron-4 15B","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":97.55,"scoreLo":89.79,"scoreHi":103.89,"reasoningOffset":0.0,"rankEligible":true,"rankMean":738.0467,"rankLo":704.0,"rankHi":773.0},{"stateId":"nemotron-4-340b-base::default","configSlug":"nemotron-4-340b-base","modelId":"nemotron-4-340b-base","modelSlug":"nemotron-4-340b-base","name":"Nemotron-4 340B Base","modelName":"Nemotron-4 340B Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":123.84,"scoreLo":117.61,"scoreHi":130.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":451.8347,"rankLo":373.0,"rankHi":528.0},{"stateId":"nemotron-4-340b-instruct::default","configSlug":"nemotron-4-340b-instruct","modelId":"nemotron-4-340b-instruct","modelSlug":"nemotron-4-340b-instruct","name":"Nemotron-4 340B Instruct","modelName":"Nemotron-4 340B Instruct","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":118.7,"scoreLo":113.52,"scoreHi":123.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":517.9236,"rankLo":457.0,"rankHi":569.0},{"stateId":"nemotron-cascade-14b-thinking::default","configSlug":"nemotron-cascade-14b-thinking","modelId":"nemotron-cascade-14b-thinking","modelSlug":"nemotron-cascade-14b-thinking","name":"Nemotron Cascade 14B Thinking","modelName":"Nemotron Cascade 14B Thinking","organizationSlug":"nvidia","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":135.09,"scoreLo":131.36,"scoreHi":138.92,"reasoningOffset":2.4,"rankEligible":true,"rankMean":306.9625,"rankLo":259.975,"rankHi":354.0},{"stateId":"nemotron-cascade-2-30b-a3b::default","configSlug":"nemotron-cascade-2-30b-a3b","modelId":"nemotron-cascade-2-30b-a3b","modelSlug":"nemotron-cascade-2-30b-a3b","name":"Nemotron Cascade 2 30B A3B","modelName":"Nemotron Cascade 2 30B A3B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":140.12,"scoreLo":137.8,"scoreHi":142.63,"reasoningOffset":0.0,"rankEligible":true,"rankMean":238.4533,"rankLo":215.0,"rankHi":265.0},{"stateId":"nemotron-cascade-8b::default","configSlug":"nemotron-cascade-8b","modelId":"nemotron-cascade-8b","modelSlug":"nemotron-cascade-8b","name":"Nemotron Cascade 8B","modelName":"Nemotron Cascade 8B","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":135.82,"scoreLo":131.86,"scoreHi":139.76,"reasoningOffset":0.0,"rankEligible":true,"rankMean":297.0039,"rankLo":245.0,"rankHi":347.0},{"stateId":"nemotron-cascade-8b-thinking::default","configSlug":"nemotron-cascade-8b-thinking","modelId":"nemotron-cascade-8b-thinking","modelSlug":"nemotron-cascade-8b-thinking","name":"Nemotron Cascade 8B Thinking","modelName":"Nemotron Cascade 8B Thinking","organizationSlug":"nvidia","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":133.72,"scoreLo":129.65,"scoreHi":137.79,"reasoningOffset":2.4,"rankEligible":true,"rankMean":325.4597,"rankLo":272.975,"rankHi":376.0},{"stateId":"nemotron-h-47b-base-8k::default","configSlug":"nemotron-h-47b-base-8k","modelId":"nemotron-h-47b-base-8k","modelSlug":"nemotron-h-47b-base-8k","name":"Nemotron-H 47B Base 8K","modelName":"Nemotron-H 47B Base 8K","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":121.06,"scoreLo":115.66,"scoreHi":126.19,"reasoningOffset":0.0,"rankEligible":true,"rankMean":488.0878,"rankLo":426.0,"rankHi":544.0},{"stateId":"nemotron-h-47b-reasoning-128k::enabled","configSlug":"nemotron-h-47b-reasoning-128k-thinking","modelId":"nemotron-h-47b-reasoning-128k","modelSlug":"nemotron-h-47b-reasoning-128k","name":"Nemotron-H 47B Reasoning 128K (thinking)","modelName":"Nemotron-H 47B Reasoning 128K","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":133.37,"scoreLo":128.12,"scoreHi":138.65,"reasoningOffset":1.07,"rankEligible":true,"rankMean":330.0022,"rankLo":259.975,"rankHi":399.0},{"stateId":"nemotron-h-4b-base-8k::default","configSlug":"nemotron-h-4b-base-8k","modelId":"nemotron-h-4b-base-8k","modelSlug":"nemotron-h-4b-base-8k","name":"Nemotron-H 4B Base 8K","modelName":"Nemotron-H 4B Base 8K","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":100.4,"scoreLo":91.99,"scoreHi":107.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":715.1322,"rankLo":669.975,"rankHi":758.0},{"stateId":"nemotron-h-4b-instruct-128k::default","configSlug":"nemotron-h-4b-instruct-128k","modelId":"nemotron-h-4b-instruct-128k","modelSlug":"nemotron-h-4b-instruct-128k","name":"Nemotron-H 4B Instruct 128K","modelName":"Nemotron-H 4B Instruct 128K","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":112.74,"scoreLo":105.88,"scoreHi":119.59,"reasoningOffset":0.0,"rankEligible":true,"rankMean":589.8261,"rankLo":516.0,"rankHi":657.0},{"stateId":"nemotron-h-56b-base-8k::default","configSlug":"nemotron-h-56b-base-8k","modelId":"nemotron-h-56b-base-8k","modelSlug":"nemotron-h-56b-base-8k","name":"Nemotron-H 56B Base 8K","modelName":"Nemotron-H 56B Base 8K","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":121.72,"scoreLo":116.44,"scoreHi":126.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":479.2358,"rankLo":419.0,"rankHi":536.025},{"stateId":"nemotron-h-8b-base-8k::default","configSlug":"nemotron-h-8b-base-8k","modelId":"nemotron-h-8b-base-8k","modelSlug":"nemotron-h-8b-base-8k","name":"Nemotron-H 8B Base 8K","modelName":"Nemotron-H 8B Base 8K","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":110.09,"scoreLo":104.1,"scoreHi":115.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":621.7136,"rankLo":567.975,"rankHi":668.0},{"stateId":"nemotron-h-8b-reasoning-128k::enabled","configSlug":"nemotron-h-8b-reasoning-128k-thinking","modelId":"nemotron-h-8b-reasoning-128k","modelSlug":"nemotron-h-8b-reasoning-128k","name":"Nemotron-H 8B Reasoning 128K (thinking)","modelName":"Nemotron-H 8B Reasoning 128K","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":127.29,"scoreLo":121.38,"scoreHi":132.78,"reasoningOffset":1.07,"rankEligible":true,"rankMean":408.2256,"rankLo":341.0,"rankHi":480.025},{"stateId":"nemotron-nano-12b-v2-base::default","configSlug":"nemotron-nano-12b-v2-base","modelId":"nemotron-nano-12b-v2-base","modelSlug":"nemotron-nano-12b-v2-base","name":"Nemotron Nano 12B V2 Base","modelName":"Nemotron Nano 12B V2 Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":120.0,"scoreLo":115.4,"scoreHi":124.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":501.9608,"rankLo":450.0,"rankHi":545.0},{"stateId":"nemotron-nano-12b-v2::default","configSlug":"nemotron-nano-12b-v2-medmarks","modelId":"nemotron-nano-12b-v2","modelSlug":"nemotron-nano-12b-v2","name":"Nemotron Nano 12B V2","modelName":"Nemotron Nano 12B V2","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":2,"score":131.67,"scoreLo":125.16,"scoreHi":138.22,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"nemotron-nano-12b-v2::disabled","configSlug":"nemotron-nano-12b-v2-no-thinking","modelId":"nemotron-nano-12b-v2","modelSlug":"nemotron-nano-12b-v2","name":"Nemotron Nano 12B V2 (disabled)","modelName":"Nemotron Nano 12B V2","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":127.81,"scoreLo":118.53,"scoreHi":136.85,"reasoningOffset":-3.85,"rankEligible":false},{"stateId":"nemotron-nano-12b-v2::enabled","configSlug":"nemotron-nano-12b-v2-thinking","modelId":"nemotron-nano-12b-v2","modelSlug":"nemotron-nano-12b-v2","name":"Nemotron Nano 12B V2 (thinking)","modelName":"Nemotron Nano 12B V2","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":133.82,"scoreLo":128.55,"scoreHi":139.12,"reasoningOffset":2.15,"rankEligible":true,"rankMean":324.0117,"rankLo":256.0,"rankHi":391.025},{"stateId":"nemotron-nano-12b-v2-vl-bf16::default","configSlug":"nemotron-nano-12b-v2-vl-bf16","modelId":"nemotron-nano-12b-v2-vl-bf16","modelSlug":"nemotron-nano-12b-v2-vl-bf16","name":"Nemotron Nano 12B V2 VL BF16","modelName":"Nemotron Nano 12B V2 VL BF16","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":126.61,"scoreLo":122.19,"scoreHi":130.56,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nemotron-nano-12b-v2-vl-bf16::disabled","configSlug":"nemotron-nano-12b-v2-vl-bf16-no-thinking","modelId":"nemotron-nano-12b-v2-vl-bf16","modelSlug":"nemotron-nano-12b-v2-vl-bf16","name":"Nemotron Nano 12B V2 VL BF16 (disabled)","modelName":"Nemotron Nano 12B V2 VL BF16","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":16,"score":125.24,"scoreLo":121.73,"scoreHi":128.34,"reasoningOffset":-1.37,"rankEligible":true},{"stateId":"nemotron-nano-12b-v2-vl-bf16::enabled","configSlug":"nemotron-nano-12b-v2-vl-bf16-thinking","modelId":"nemotron-nano-12b-v2-vl-bf16","modelSlug":"nemotron-nano-12b-v2-vl-bf16","name":"Nemotron Nano 12B V2 VL BF16 (thinking)","modelName":"Nemotron Nano 12B V2 VL BF16","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":18,"score":128.2,"scoreLo":124.7,"scoreHi":131.34,"reasoningOffset":1.6,"rankEligible":true,"rankMean":397.4625,"rankLo":360.975,"rankHi":432.0},{"stateId":"nemotron-nano-12b-v2-vl-fp8::default","configSlug":"nemotron-nano-12b-v2-vl-fp8","modelId":"nemotron-nano-12b-v2-vl-fp8","modelSlug":"nemotron-nano-12b-v2-vl-fp8","name":"Nemotron Nano 12B V2 VL FP8","modelName":"Nemotron Nano 12B V2 VL FP8","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":127.02,"scoreLo":121.76,"scoreHi":131.9,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nemotron-nano-12b-v2-vl-fp8::disabled","configSlug":"nemotron-nano-12b-v2-vl-fp8-no-thinking","modelId":"nemotron-nano-12b-v2-vl-fp8","modelSlug":"nemotron-nano-12b-v2-vl-fp8","name":"Nemotron Nano 12B V2 VL FP8 (disabled)","modelName":"Nemotron Nano 12B V2 VL FP8","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":124.3,"scoreLo":119.88,"scoreHi":128.33,"reasoningOffset":-2.72,"rankEligible":true},{"stateId":"nemotron-nano-12b-v2-vl-fp8::enabled","configSlug":"nemotron-nano-12b-v2-vl-fp8-thinking","modelId":"nemotron-nano-12b-v2-vl-fp8","modelSlug":"nemotron-nano-12b-v2-vl-fp8","name":"Nemotron Nano 12B V2 VL FP8 (thinking)","modelName":"Nemotron Nano 12B V2 VL FP8","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":8,"score":129.94,"scoreLo":125.86,"scoreHi":133.82,"reasoningOffset":2.91,"rankEligible":true,"rankMean":375.3958,"rankLo":329.0,"rankHi":419.0},{"stateId":"nemotron-nano-9b-v2::default","configSlug":"nemotron-nano-9b-v2","modelId":"nemotron-nano-9b-v2","modelSlug":"nemotron-nano-9b-v2","name":"Nemotron Nano 9B V2","modelName":"Nemotron Nano 9B V2","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":125.44,"scoreLo":121.04,"scoreHi":129.37,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nemotron-nano-9b-v2-base::default","configSlug":"nemotron-nano-9b-v2-base","modelId":"nemotron-nano-9b-v2-base","modelSlug":"nemotron-nano-9b-v2-base","name":"Nemotron Nano 9B V2 Base","modelName":"Nemotron Nano 9B V2 Base","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":116.75,"scoreLo":111.95,"scoreHi":121.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":541.61,"rankLo":499.0,"rankHi":583.025},{"stateId":"nemotron-nano-9b-v2::disabled","configSlug":"nemotron-nano-9b-v2-no-thinking","modelId":"nemotron-nano-9b-v2","modelSlug":"nemotron-nano-9b-v2","name":"Nemotron Nano 9B V2 (disabled)","modelName":"Nemotron Nano 9B V2","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":127.84,"scoreLo":123.97,"scoreHi":131.45,"reasoningOffset":2.4,"rankEligible":true},{"stateId":"nemotron-nano-9b-v2::enabled","configSlug":"nemotron-nano-9b-v2-thinking","modelId":"nemotron-nano-9b-v2","modelSlug":"nemotron-nano-9b-v2","name":"Nemotron Nano 9B V2 (thinking)","modelName":"Nemotron Nano 9B V2","organizationSlug":"nvidia","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":13,"score":128.82,"scoreLo":125.44,"scoreHi":132.07,"reasoningOffset":3.38,"rankEligible":true,"rankMean":389.9669,"rankLo":353.0,"rankHi":423.0},{"stateId":"neural-chat-7b-v3-1::default","configSlug":"neural-chat-7b-v3-1","modelId":"neural-chat-7b-v3-1","modelSlug":"neural-chat-7b-v3-1","name":"Neural Chat 7B v3-1","modelName":"Neural Chat 7B v3-1","organizationSlug":"intel","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":99.78,"scoreLo":92.77,"scoreHi":105.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":720.2461,"rankLo":692.0,"rankHi":749.025},{"stateId":"nex-n2-mini::default","configSlug":"nex-n2-mini","modelId":"nex-n2-mini","modelSlug":"nex-n2-mini","name":"Nex-N2-mini","modelName":"Nex-N2-mini","organizationSlug":"nex-agi","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":146.04,"scoreLo":137.26,"scoreHi":154.59,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"nex-n2-mini::effort:minimal","configSlug":"nex-n2-mini-minimal","modelId":"nex-n2-mini","modelSlug":"nex-n2-mini","name":"Nex-N2-mini (minimal)","modelName":"Nex-N2-mini","organizationSlug":"nex-agi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":141.61,"scoreLo":132.3,"scoreHi":150.74,"reasoningOffset":-4.42,"rankEligible":false},{"stateId":"nex-n2-mini::enabled","configSlug":"nex-n2-mini-thinking-context-arena-mrcr-v2","modelId":"nex-n2-mini","modelSlug":"nex-n2-mini","name":"Nex-N2-mini (thinking)","modelName":"Nex-N2-mini","organizationSlug":"nex-agi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":147.34,"scoreLo":138.54,"scoreHi":156.01,"reasoningOffset":1.31,"rankEligible":false},{"stateId":"nex-n2-pro::default","configSlug":"nex-n2-pro","modelId":"nex-n2-pro","modelSlug":"nex-n2-pro","name":"Nex-N2-Pro","modelName":"Nex-N2-Pro","organizationSlug":"nex-agi","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":14,"score":153.52,"scoreLo":150.3,"scoreHi":157.27,"reasoningOffset":0.0,"rankEligible":true,"rankMean":97.4139,"rankLo":80.0,"rankHi":118.0},{"stateId":"north-mini-code-1-0::default","configSlug":"north-mini-code-1-0","modelId":"north-mini-code-1-0","modelSlug":"north-mini-code-1-0","name":"North Mini Code","modelName":"North Mini Code","organizationSlug":"cohere","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":140.43,"scoreLo":137.5,"scoreHi":143.45,"reasoningOffset":0.0,"rankEligible":true,"rankMean":234.9722,"rankLo":207.0,"rankHi":271.0},{"stateId":"nous-hermes-2-mixtral-8x7b-sft::default","configSlug":"nous-hermes-2-mixtral-8x7b-sft","modelId":"nous-hermes-2-mixtral-8x7b-sft","modelSlug":"nous-hermes-2-mixtral-8x7b-sft","name":"Nous Hermes 2 Mixtral 8x7B SFT","modelName":"Nous Hermes 2 Mixtral 8x7B SFT","organizationSlug":"nous-research","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":105.6,"scoreLo":99.95,"scoreHi":110.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":670.6542,"rankLo":637.0,"rankHi":700.0},{"stateId":"nous-hermes-2-yi-34b::default","configSlug":"nous-hermes-2-yi-34b","modelId":"nous-hermes-2-yi-34b","modelSlug":"nous-hermes-2-yi-34b","name":"Nous Hermes 2 Yi 34B","modelName":"Nous Hermes 2 Yi 34B","organizationSlug":"nous-research","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":111.73,"scoreLo":106.12,"scoreHi":116.39,"reasoningOffset":0.0,"rankEligible":true,"rankMean":602.1919,"rankLo":557.0,"rankHi":646.0},{"stateId":"nova-2-lite-v1-0::default","configSlug":"nova-2-lite-v1-0","modelId":"nova-2-lite-v1-0","modelSlug":"nova-2-lite-v1-0","name":"Nova 2 Lite","modelName":"Nova 2 Lite","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":137.12,"scoreLo":134.35,"scoreHi":139.89,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"nova-2-lite-v1-0::effort:high","configSlug":"nova-2-lite-v1-0-high","modelId":"nova-2-lite-v1-0","modelSlug":"nova-2-lite-v1-0","name":"Nova 2 Lite (high)","modelName":"Nova 2 Lite","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":12,"score":139.03,"scoreLo":136.48,"scoreHi":141.75,"reasoningOffset":1.91,"rankEligible":true,"rankMean":253.0711,"rankLo":223.0,"rankHi":284.0},{"stateId":"nova-2-lite-v1-0::effort:low","configSlug":"nova-2-lite-v1-0-low","modelId":"nova-2-lite-v1-0","modelSlug":"nova-2-lite-v1-0","name":"Nova 2 Lite (low)","modelName":"Nova 2 Lite","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":9,"score":136.47,"scoreLo":133.6,"scoreHi":139.21,"reasoningOffset":-0.65,"rankEligible":true},{"stateId":"nova-2-lite-v1-0::effort:medium","configSlug":"nova-2-lite-v1-0-medium","modelId":"nova-2-lite-v1-0","modelSlug":"nova-2-lite-v1-0","name":"Nova 2 Lite (medium)","modelName":"Nova 2 Lite","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":10,"score":138.56,"scoreLo":135.82,"scoreHi":141.25,"reasoningOffset":1.43,"rankEligible":true},{"stateId":"nova-2-lite-v1-0::disabled","configSlug":"nova-2-lite-v1-0-no-thinking","modelId":"nova-2-lite-v1-0","modelSlug":"nova-2-lite-v1-0","name":"Nova 2 Lite (disabled)","modelName":"Nova 2 Lite","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":11,"score":131.67,"scoreLo":128.7,"scoreHi":134.51,"reasoningOffset":-5.46,"rankEligible":true},{"stateId":"nova-2-omni-v1-0::effort:low","configSlug":"nova-2-omni-v1-0-low","modelId":"nova-2-omni-v1-0","modelSlug":"nova-2-omni-v1-0","name":"Nova 2 Omni (Preview) (low)","modelName":"Nova 2 Omni (Preview)","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":9,"score":135.95,"scoreLo":133.15,"scoreHi":138.82,"reasoningOffset":0.84,"rankEligible":true},{"stateId":"nova-2-omni-v1-0::effort:medium","configSlug":"nova-2-omni-v1-0-medium","modelId":"nova-2-omni-v1-0","modelSlug":"nova-2-omni-v1-0","name":"Nova 2 Omni (Preview) (medium)","modelName":"Nova 2 Omni (Preview)","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":9,"score":137.18,"scoreLo":134.3,"scoreHi":140.13,"reasoningOffset":2.07,"rankEligible":true,"rankMean":278.3042,"rankLo":242.0,"rankHi":312.0},{"stateId":"nova-2-omni-v1-0::disabled","configSlug":"nova-2-omni-v1-0-no-thinking","modelId":"nova-2-omni-v1-0","modelSlug":"nova-2-omni-v1-0","name":"Nova 2 Omni (Preview) (disabled)","modelName":"Nova 2 Omni (Preview)","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":9,"score":131.11,"scoreLo":127.93,"scoreHi":134.08,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"nova-2-pro-preview-20251202-v1-0::effort:high","configSlug":"nova-2-pro-preview-20251202-v1-0-high","modelId":"nova-2-pro-preview-20251202-v1-0","modelSlug":"nova-2-pro-preview-20251202-v1-0","name":"Nova 2 Pro (Preview) (high)","modelName":"Nova 2 Pro (Preview)","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":140.86,"scoreLo":137.24,"scoreHi":144.61,"reasoningOffset":0.52,"rankEligible":false},{"stateId":"nova-2-pro-preview-20251202-v1-0::effort:low","configSlug":"nova-2-pro-preview-20251202-v1-0-low","modelId":"nova-2-pro-preview-20251202-v1-0","modelSlug":"nova-2-pro-preview-20251202-v1-0","name":"Nova 2 Pro (Preview) (low)","modelName":"Nova 2 Pro (Preview)","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":12,"score":140.93,"scoreLo":138.3,"scoreHi":143.6,"reasoningOffset":0.59,"rankEligible":true},{"stateId":"nova-2-pro-preview-20251202-v1-0::effort:medium","configSlug":"nova-2-pro-preview-20251202-v1-0-medium","modelId":"nova-2-pro-preview-20251202-v1-0","modelSlug":"nova-2-pro-preview-20251202-v1-0","name":"Nova 2 Pro (Preview) (medium)","modelName":"Nova 2 Pro (Preview)","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":12,"score":142.01,"scoreLo":139.41,"scoreHi":144.82,"reasoningOffset":1.68,"rankEligible":true,"rankMean":216.5375,"rankLo":194.0,"rankHi":243.0},{"stateId":"nova-2-pro-preview-20251202-v1-0::disabled","configSlug":"nova-2-pro-preview-20251202-v1-0-no-thinking","modelId":"nova-2-pro-preview-20251202-v1-0","modelSlug":"nova-2-pro-preview-20251202-v1-0","name":"Nova 2 Pro (Preview) (disabled)","modelName":"Nova 2 Pro (Preview)","organizationSlug":"amazon","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":12,"score":136.34,"scoreLo":133.8,"scoreHi":138.92,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"nova-lite-v1-0::default","configSlug":"nova-lite-v1-0","modelId":"nova-lite-v1-0","modelSlug":"nova-lite-v1-0","name":"Nova Lite","modelName":"Nova Lite","organizationSlug":"amazon","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":117.78,"scoreLo":113.59,"scoreHi":121.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":529.7553,"rankLo":495.0,"rankHi":563.0},{"stateId":"nova-micro-v1-0::default","configSlug":"nova-micro-v1-0","modelId":"nova-micro-v1-0","modelSlug":"nova-micro-v1-0","name":"Nova Micro","modelName":"Nova Micro","organizationSlug":"amazon","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":113.77,"scoreLo":108.56,"scoreHi":118.22,"reasoningOffset":0.0,"rankEligible":true,"rankMean":577.1419,"rankLo":537.0,"rankHi":621.0},{"stateId":"nova-premier-v1-0::default","configSlug":"nova-premier-v1-0","modelId":"nova-premier-v1-0","modelSlug":"nova-premier-v1-0","name":"Nova Premier","modelName":"Nova Premier","organizationSlug":"amazon","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":131.26,"scoreLo":128.03,"scoreHi":134.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":358.2239,"rankLo":320.0,"rankHi":397.0},{"stateId":"nova-pro-v1-0::default","configSlug":"nova-pro-v1-0","modelId":"nova-pro-v1-0","modelSlug":"nova-pro-v1-0","name":"Nova Pro","modelName":"Nova Pro","organizationSlug":"amazon","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":33,"score":123.51,"scoreLo":120.32,"scoreHi":126.53,"reasoningOffset":0.0,"rankEligible":true,"rankMean":454.8158,"rankLo":426.0,"rankHi":488.0},{"stateId":"nvlm-d-72b::default","configSlug":"nvlm-d-72b","modelId":"nvlm-d-72b","modelSlug":"nvlm-d-72b","name":"NVLM-D-72B","modelName":"NVLM-D-72B","organizationSlug":"nvidia","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":120.08,"scoreLo":115.73,"scoreHi":124.05,"reasoningOffset":0.0,"rankEligible":true,"rankMean":501.1183,"rankLo":454.0,"rankHi":541.0},{"stateId":"o1::default","configSlug":"o1","modelId":"o1","modelSlug":"o1","name":"o1 (2024-12-17)","modelName":"o1 (2024-12-17)","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":41,"score":140.8,"scoreLo":138.78,"scoreHi":142.97,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"o1::enabled","configSlug":"o1-aa-tau2-bench-telecom","modelId":"o1","modelSlug":"o1","name":"o1 (2024-12-17) (enabled)","modelName":"o1 (2024-12-17)","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":141.55,"scoreLo":138.05,"scoreHi":145.27,"reasoningOffset":0.74,"rankEligible":false},{"stateId":"o1::effort:high","configSlug":"o1-high","modelId":"o1","modelSlug":"o1","name":"o1 (2024-12-17) (high)","modelName":"o1 (2024-12-17)","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":35,"score":141.28,"scoreLo":139.39,"scoreHi":143.36,"reasoningOffset":0.48,"rankEligible":true,"rankMean":223.5944,"rankLo":210.0,"rankHi":241.0},{"stateId":"o1::effort:low","configSlug":"o1-low","modelId":"o1","modelSlug":"o1","name":"o1 (2024-12-17) (low)","modelName":"o1 (2024-12-17)","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":140.45,"scoreLo":137.07,"scoreHi":144.07,"reasoningOffset":-0.35,"rankEligible":false},{"stateId":"o1::effort:medium","configSlug":"o1-medium","modelId":"o1","modelSlug":"o1","name":"o1 (2024-12-17) (medium)","modelName":"o1 (2024-12-17)","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":12,"score":140.96,"scoreLo":138.77,"scoreHi":143.29,"reasoningOffset":0.16,"rankEligible":true},{"stateId":"o1-mini::default","configSlug":"o1-mini","modelId":"o1-mini","modelSlug":"o1-mini","name":"o1-mini","modelName":"o1-mini","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":39,"score":132.14,"scoreLo":129.73,"scoreHi":134.55,"reasoningOffset":2.4,"rankEligible":true,"rankMean":346.6514,"rankLo":321.0,"rankHi":373.0},{"stateId":"o1-mini::effort:high","configSlug":"o1-mini-high","modelId":"o1-mini","modelSlug":"o1-mini","name":"o1-mini (high)","modelName":"o1-mini","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":131.08,"scoreLo":128.08,"scoreHi":133.99,"reasoningOffset":1.34,"rankEligible":false},{"stateId":"o1-mini::effort:medium","configSlug":"o1-mini-medium","modelId":"o1-mini","modelSlug":"o1-mini","name":"o1-mini (medium)","modelName":"o1-mini","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":129.66,"scoreLo":125.99,"scoreHi":133.04,"reasoningOffset":-0.08,"rankEligible":false},{"stateId":"o1-preview::default","configSlug":"o1-preview","modelId":"o1-preview","modelSlug":"o1-preview","name":"o1 Preview","modelName":"o1 Preview","organizationSlug":"openai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":32,"score":137.93,"scoreLo":135.18,"scoreHi":140.67,"reasoningOffset":2.4,"rankEligible":true,"rankMean":268.1853,"rankLo":235.0,"rankHi":301.0},{"stateId":"o1-pro::default","configSlug":"o1-pro","modelId":"o1-pro","modelSlug":"o1-pro","name":"o1-pro","modelName":"o1-pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":140.59,"scoreLo":137.26,"scoreHi":144.11,"reasoningOffset":0.0,"rankEligible":true,"rankMean":233.2267,"rankLo":200.0,"rankHi":274.0},{"stateId":"o1-pro::effort:high","configSlug":"o1-pro-high","modelId":"o1-pro","modelSlug":"o1-pro","name":"o1-pro (high)","modelName":"o1-pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":140.22,"scoreLo":136.45,"scoreHi":144.15,"reasoningOffset":-0.37,"rankEligible":false},{"stateId":"o1-pro::effort:medium","configSlug":"o1-pro-medium","modelId":"o1-pro","modelSlug":"o1-pro","name":"o1-pro (medium)","modelName":"o1-pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":141.64,"scoreLo":138.17,"scoreHi":145.16,"reasoningOffset":1.05,"rankEligible":false},{"stateId":"o3::default","configSlug":"o3","modelId":"o3","modelSlug":"o3","name":"o3","modelName":"o3","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":77,"score":147.12,"scoreLo":145.08,"scoreHi":149.44,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"o3::enabled","configSlug":"o3-aa-tau2-bench-telecom","modelId":"o3","modelSlug":"o3","name":"o3 (enabled)","modelName":"o3","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":9,"score":146.95,"scoreLo":143.73,"scoreHi":150.43,"reasoningOffset":-0.17,"rankEligible":true},{"stateId":"o3::effort:high","configSlug":"o3-high","modelId":"o3","modelSlug":"o3","name":"o3 (high)","modelName":"o3","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":79,"score":148.31,"scoreLo":146.21,"scoreHi":150.7,"reasoningOffset":1.19,"rankEligible":true,"rankMean":150.0744,"rankLo":140.975,"rankHi":161.0},{"stateId":"o3::effort:low","configSlug":"o3-low","modelId":"o3","modelSlug":"o3","name":"o3 (low)","modelName":"o3","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":17,"score":145.82,"scoreLo":143.76,"scoreHi":148.05,"reasoningOffset":-1.31,"rankEligible":true},{"stateId":"o3::effort:medium","configSlug":"o3-medium","modelId":"o3","modelSlug":"o3","name":"o3 (medium)","modelName":"o3","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":40,"score":147.41,"scoreLo":145.32,"scoreHi":149.74,"reasoningOffset":0.29,"rankEligible":true},{"stateId":"o3-mini::default","configSlug":"o3-mini","modelId":"o3-mini","modelSlug":"o3-mini","name":"o3-mini","modelName":"o3-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":136.58,"scoreLo":134.45,"scoreHi":138.75,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"o3-mini::enabled","configSlug":"o3-mini-forecastbench-zero-shot","modelId":"o3-mini","modelSlug":"o3-mini","name":"o3-mini (enabled)","modelName":"o3-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":3,"score":138.04,"scoreLo":134.47,"scoreHi":141.7,"reasoningOffset":1.47,"rankEligible":false},{"stateId":"o3-mini::effort:high","configSlug":"o3-mini-high","modelId":"o3-mini","modelSlug":"o3-mini","name":"o3-mini (high)","modelName":"o3-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":77,"score":138.8,"scoreLo":137.07,"scoreHi":140.62,"reasoningOffset":2.22,"rankEligible":true,"rankMean":256.1206,"rankLo":240.0,"rankHi":273.0},{"stateId":"o3-mini::effort:low","configSlug":"o3-mini-low","modelId":"o3-mini","modelSlug":"o3-mini","name":"o3-mini (low)","modelName":"o3-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":20,"score":136.26,"scoreLo":134.36,"scoreHi":138.11,"reasoningOffset":-0.32,"rankEligible":true},{"stateId":"o3-mini::effort:medium","configSlug":"o3-mini-medium","modelId":"o3-mini","modelSlug":"o3-mini","name":"o3-mini (medium)","modelName":"o3-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":41,"score":138.19,"scoreLo":136.41,"scoreHi":140.01,"reasoningOffset":1.61,"rankEligible":true},{"stateId":"o3-mini::effort:minimal","configSlug":"o3-mini-minimal","modelId":"o3-mini","modelSlug":"o3-mini","name":"o3-mini (minimal)","modelName":"o3-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":132.2,"scoreLo":127.36,"scoreHi":137.08,"reasoningOffset":-4.38,"rankEligible":false},{"stateId":"o3::effort:minimal","configSlug":"o3-minimal","modelId":"o3","modelSlug":"o3","name":"o3 (minimal)","modelName":"o3","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":143.07,"scoreLo":138.33,"scoreHi":147.98,"reasoningOffset":-4.06,"rankEligible":false},{"stateId":"o3-pro::default","configSlug":"o3-pro","modelId":"o3-pro","modelSlug":"o3-pro","name":"o3-pro","modelName":"o3-pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":148.66,"scoreLo":145.42,"scoreHi":152.17,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"o3-pro::effort:high","configSlug":"o3-pro-high","modelId":"o3-pro","modelSlug":"o3-pro","name":"o3-pro (high)","modelName":"o3-pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":9,"score":148.9,"scoreLo":146.16,"scoreHi":151.85,"reasoningOffset":0.24,"rankEligible":true,"rankMean":143.6564,"rankLo":124.0,"rankHi":164.0},{"stateId":"o3-pro::effort:low","configSlug":"o3-pro-low","modelId":"o3-pro","modelSlug":"o3-pro","name":"o3-pro (low)","modelName":"o3-pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":147.01,"scoreLo":144.25,"scoreHi":149.95,"reasoningOffset":-1.65,"rankEligible":false},{"stateId":"o3-pro::effort:medium","configSlug":"o3-pro-medium","modelId":"o3-pro","modelSlug":"o3-pro","name":"o3-pro (medium)","modelName":"o3-pro","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":4,"score":148.15,"scoreLo":145.38,"scoreHi":151.02,"reasoningOffset":-0.51,"rankEligible":true},{"stateId":"o4-mini::default","configSlug":"o4-mini","modelId":"o4-mini","modelSlug":"o4-mini","name":"o4-mini","modelName":"o4-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":44,"score":141.76,"scoreLo":139.92,"scoreHi":143.82,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"o4-mini::enabled","configSlug":"o4-mini-forecastbench-zero-shot","modelId":"o4-mini","modelSlug":"o4-mini","name":"o4-mini (enabled)","modelName":"o4-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":4,"score":142.63,"scoreLo":139.3,"scoreHi":146.13,"reasoningOffset":0.87,"rankEligible":true},{"stateId":"o4-mini::effort:high","configSlug":"o4-mini-high","modelId":"o4-mini","modelSlug":"o4-mini","name":"o4-mini (high)","modelName":"o4-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":83,"score":143.86,"scoreLo":142.05,"scoreHi":145.9,"reasoningOffset":2.1,"rankEligible":true,"rankMean":198.2953,"rankLo":187.0,"rankHi":209.0},{"stateId":"o4-mini::effort:low","configSlug":"o4-mini-low","modelId":"o4-mini","modelSlug":"o4-mini","name":"o4-mini (low)","modelName":"o4-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":22,"score":140.22,"scoreLo":138.44,"scoreHi":142.14,"reasoningOffset":-1.54,"rankEligible":true},{"stateId":"o4-mini::effort:medium","configSlug":"o4-mini-medium","modelId":"o4-mini","modelSlug":"o4-mini","name":"o4-mini (medium)","modelName":"o4-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":37,"score":142.05,"scoreLo":140.31,"scoreHi":143.95,"reasoningOffset":0.29,"rankEligible":true},{"stateId":"o4-mini::effort:minimal","configSlug":"o4-mini-minimal","modelId":"o4-mini","modelSlug":"o4-mini","name":"o4-mini (minimal)","modelName":"o4-mini","organizationSlug":"openai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":137.33,"scoreLo":132.88,"scoreHi":142.03,"reasoningOffset":-4.44,"rankEligible":false},{"stateId":"olmo-2-0325-32b-instruct::default","configSlug":"olmo-2-0325-32b-instruct","modelId":"olmo-2-0325-32b-instruct","modelSlug":"olmo-2-0325-32b-instruct","name":"OLMo 2 32B Instruct","modelName":"OLMo 2 32B Instruct","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":111.27,"scoreLo":105.26,"scoreHi":116.49,"reasoningOffset":0.0,"rankEligible":true,"rankMean":607.7894,"rankLo":555.0,"rankHi":659.025},{"stateId":"olmo-2-1124-13b-instruct::default","configSlug":"olmo-2-1124-13b-instruct-helm-capabilities","modelId":"olmo-2-1124-13b-instruct","modelSlug":"olmo-2-1124-13b-instruct","name":"OLMo 2 1124 13B Instruct","modelName":"OLMo 2 1124 13B Instruct","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":5,"score":101.79,"scoreLo":93.66,"scoreHi":109.08,"reasoningOffset":0.0,"rankEligible":true,"rankMean":703.4233,"rankLo":649.975,"rankHi":756.0},{"stateId":"olmo-2-1124-7b-instruct::default","configSlug":"olmo-2-1124-7b-instruct","modelId":"olmo-2-1124-7b-instruct","modelSlug":"olmo-2-1124-7b-instruct","name":"OLMo 2 1124 7B Instruct","modelName":"OLMo 2 1124 7B Instruct","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":94.73,"scoreLo":86.0,"scoreHi":102.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":758.8806,"rankLo":715.0,"rankHi":799.0},{"stateId":"olmo-3-1-32b-instruct::default","configSlug":"olmo-3-1-32b-instruct","modelId":"olmo-3-1-32b-instruct","modelSlug":"olmo-3-1-32b-instruct","name":"Olmo 3.1 32B Instruct","modelName":"Olmo 3.1 32B Instruct","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":121.35,"scoreLo":113.54,"scoreHi":128.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":483.7072,"rankLo":397.975,"rankHi":572.0},{"stateId":"olmo-3-1-32b-think::default","configSlug":"olmo-3-1-32b-think","modelId":"olmo-3-1-32b-think","modelSlug":"olmo-3-1-32b-think","name":"Olmo 3.1 32B Think","modelName":"Olmo 3.1 32B Think","organizationSlug":"allen-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":14,"score":130.73,"scoreLo":126.55,"scoreHi":134.84,"reasoningOffset":2.4,"rankEligible":true,"rankMean":365.0725,"rankLo":314.0,"rankHi":411.0},{"stateId":"olmo-3-32b-think::default","configSlug":"olmo-3-32b-think","modelId":"olmo-3-32b-think","modelSlug":"olmo-3-32b-think","name":"Olmo 3 32B Think","modelName":"Olmo 3 32B Think","organizationSlug":"allen-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":12,"score":128.83,"scoreLo":124.36,"scoreHi":132.93,"reasoningOffset":2.4,"rankEligible":true,"rankMean":389.3608,"rankLo":339.0,"rankHi":439.0},{"stateId":"olmo-3-7b-instruct::default","configSlug":"olmo-3-7b-instruct","modelId":"olmo-3-7b-instruct","modelSlug":"olmo-3-7b-instruct","name":"Olmo 3 7B Instruct","modelName":"Olmo 3 7B Instruct","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":110.73,"scoreLo":103.1,"scoreHi":117.36,"reasoningOffset":0.0,"rankEligible":true,"rankMean":613.5078,"rankLo":543.0,"rankHi":680.025},{"stateId":"olmo-3-7b-think::default","configSlug":"olmo-3-7b-think","modelId":"olmo-3-7b-think","modelSlug":"olmo-3-7b-think","name":"Olmo 3 7B Think","modelName":"Olmo 3 7B Think","organizationSlug":"allen-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":10,"score":122.77,"scoreLo":117.1,"scoreHi":127.93,"reasoningOffset":2.4,"rankEligible":true,"rankMean":465.2919,"rankLo":405.0,"rankHi":532.025},{"stateId":"olmoe-1b-7b-0125-instruct::default","configSlug":"olmoe-1b-7b-0125-instruct-helm-capabilities","modelId":"olmoe-1b-7b-0125-instruct","modelSlug":"olmoe-1b-7b-0125-instruct","name":"OLMoE 1B-7B 0125 Instruct","modelName":"OLMoE 1B-7B 0125 Instruct","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":false,"n":5,"score":81.45,"scoreLo":70.05,"scoreHi":91.21,"reasoningOffset":0.0,"rankEligible":true,"rankMean":827.4097,"rankLo":799.0,"rankHi":852.0},{"stateId":"open-llama-7b::default","configSlug":"open-llama-7b","modelId":"open-llama-7b","modelSlug":"open-llama-7b","name":"OpenLLaMA 7B","modelName":"OpenLLaMA 7B","organizationSlug":"openlm-research","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":72.47,"scoreLo":58.7,"scoreHi":83.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":853.9128,"rankLo":833.0,"rankHi":869.0},{"stateId":"openchat-3-5-1210::default","configSlug":"openchat-3-5-1210","modelId":"openchat-3-5-1210","modelSlug":"openchat-3-5-1210","name":"OpenChat 3.5 1210","modelName":"OpenChat 3.5 1210","organizationSlug":"openchat","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":101.57,"scoreLo":94.95,"scoreHi":107.0,"reasoningOffset":0.0,"rankEligible":true,"rankMean":705.9856,"rankLo":679.0,"rankHi":730.0},{"stateId":"opt-1-3b::default","configSlug":"opt-1-3b","modelId":"opt-1-3b","modelSlug":"opt-1-3b","name":"OPT-1.3B","modelName":"OPT-1.3B","organizationSlug":"meta","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":55.83,"scoreLo":40.91,"scoreHi":68.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":875.9225,"rankLo":869.0,"rankHi":882.0},{"stateId":"palm-2-l::default","configSlug":"palm-2-l","modelId":"palm-2-l","modelSlug":"palm-2-l","name":"PaLM 2-L","modelName":"PaLM 2-L","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":12,"score":116.09,"scoreLo":109.5,"scoreHi":122.1,"reasoningOffset":0.0,"rankEligible":true,"rankMean":549.3047,"rankLo":479.0,"rankHi":616.0},{"stateId":"palm-2-m::default","configSlug":"palm-2-m","modelId":"palm-2-m","modelSlug":"palm-2-m","name":"PaLM 2-M","modelName":"PaLM 2-M","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":108.85,"scoreLo":100.07,"scoreHi":116.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":634.1947,"rankLo":546.0,"rankHi":705.025},{"stateId":"palm-2-s::default","configSlug":"palm-2-s","modelId":"palm-2-s","modelSlug":"palm-2-s","name":"PaLM 2-S","modelName":"PaLM 2-S","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":102.62,"scoreLo":93.72,"scoreHi":110.85,"reasoningOffset":0.0,"rankEligible":true,"rankMean":695.79,"rankLo":625.0,"rankHi":753.0},{"stateId":"palm-540b::default","configSlug":"palm-540b","modelId":"palm-540b","modelSlug":"palm-540b","name":"PaLM 540B","modelName":"PaLM 540B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":101.71,"scoreLo":94.78,"scoreHi":107.8,"reasoningOffset":0.0,"rankEligible":true,"rankMean":704.6653,"rankLo":668.0,"rankHi":738.0},{"stateId":"palm-62b::default","configSlug":"palm-62b","modelId":"palm-62b","modelSlug":"palm-62b","name":"PaLM 62B","modelName":"PaLM 62B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":89.44,"scoreLo":81.04,"scoreHi":96.84,"reasoningOffset":0.0,"rankEligible":true,"rankMean":792.7514,"rankLo":767.0,"rankHi":815.0},{"stateId":"palmyra-x-004::default","configSlug":"palmyra-x-004","modelId":"palmyra-x-004","modelSlug":"palmyra-x-004","name":"Palmyra X 004","modelName":"Palmyra X 004","organizationSlug":"writer","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":122.61,"scoreLo":116.93,"scoreHi":128.09,"reasoningOffset":0.0,"rankEligible":true,"rankMean":467.4997,"rankLo":402.0,"rankHi":534.0},{"stateId":"palmyra-x5::default","configSlug":"palmyra-x5","modelId":"palmyra-x5","modelSlug":"palmyra-x5","name":"Palmyra X5","modelName":"Palmyra X5","organizationSlug":"writer","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":135.05,"scoreLo":130.36,"scoreHi":139.98,"reasoningOffset":2.4,"rankEligible":true,"rankMean":307.2492,"rankLo":244.0,"rankHi":367.0},{"stateId":"phi-1-5::default","configSlug":"phi-1-5","modelId":"phi-1-5","modelSlug":"phi-1-5","name":"Phi-1.5","modelName":"Phi-1.5","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":80.79,"scoreLo":71.6,"scoreHi":88.3,"reasoningOffset":0.0,"rankEligible":true,"rankMean":830.7278,"rankLo":820.0,"rankHi":841.0},{"stateId":"phi-2::default","configSlug":"phi-2","modelId":"phi-2","modelSlug":"phi-2","name":"Phi-2","modelName":"Phi-2","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":93.29,"scoreLo":85.79,"scoreHi":99.45,"reasoningOffset":0.0,"rankEligible":true,"rankMean":770.2883,"rankLo":750.0,"rankHi":788.0},{"stateId":"phi-3-5-mini-instruct::default","configSlug":"phi-3-5-mini-instruct","modelId":"phi-3-5-mini-instruct","modelSlug":"phi-3-5-mini-instruct","name":"Phi-3.5 Mini Instruct","modelName":"Phi-3.5 Mini Instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":31,"score":107.63,"scoreLo":102.46,"scoreHi":111.92,"reasoningOffset":0.0,"rankEligible":true,"rankMean":650.0317,"rankLo":624.0,"rankHi":674.0},{"stateId":"phi-3-5-moe-instruct::default","configSlug":"phi-3-5-moe-instruct","modelId":"phi-3-5-moe-instruct","modelSlug":"phi-3-5-moe-instruct","name":"Phi-3.5-MoE-instruct","modelName":"Phi-3.5-MoE-instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":117.83,"scoreLo":113.43,"scoreHi":121.87,"reasoningOffset":0.0,"rankEligible":true,"rankMean":529.1214,"rankLo":486.0,"rankHi":567.0},{"stateId":"phi-3-5-vision-instruct::default","configSlug":"phi-3-5-vision-instruct","modelId":"phi-3-5-vision-instruct","modelSlug":"phi-3-5-vision-instruct","name":"Phi-3.5-vision-instruct","modelName":"Phi-3.5-vision-instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":98.26,"scoreLo":91.45,"scoreHi":104.13,"reasoningOffset":0.0,"rankEligible":true,"rankMean":732.5056,"rankLo":704.0,"rankHi":763.0},{"stateId":"phi-3-medium-128k-instruct::default","configSlug":"phi-3-medium-128k-instruct","modelId":"phi-3-medium-128k-instruct","modelSlug":"phi-3-medium-128k-instruct","name":"Phi-3 Medium 128K Instruct","modelName":"Phi-3 Medium 128K Instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":113.54,"scoreLo":108.78,"scoreHi":117.83,"reasoningOffset":0.0,"rankEligible":true,"rankMean":579.7931,"rankLo":543.975,"rankHi":618.0},{"stateId":"phi-3-medium-4k-instruct::default","configSlug":"phi-3-medium-4k-instruct","modelId":"phi-3-medium-4k-instruct","modelSlug":"phi-3-medium-4k-instruct","name":"Phi-3 Medium 4K Instruct","modelName":"Phi-3 Medium 4K Instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":114.66,"scoreLo":110.27,"scoreHi":118.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":566.0572,"rankLo":535.0,"rankHi":600.0},{"stateId":"phi-3-mini-128k-instruct::default","configSlug":"phi-3-mini-128k-instruct","modelId":"phi-3-mini-128k-instruct","modelSlug":"phi-3-mini-128k-instruct","name":"Phi-3 Mini 128K Instruct","modelName":"Phi-3 Mini 128K Instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":105.83,"scoreLo":99.75,"scoreHi":110.92,"reasoningOffset":0.0,"rankEligible":true,"rankMean":668.2611,"rankLo":635.0,"rankHi":699.0},{"stateId":"phi-3-mini-4k-instruct::default","configSlug":"phi-3-mini-4k-instruct","modelId":"phi-3-mini-4k-instruct","modelSlug":"phi-3-mini-4k-instruct","name":"Phi-3-mini-4k-instruct","modelName":"Phi-3-mini-4k-instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":106.8,"scoreLo":101.45,"scoreHi":111.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":658.9042,"rankLo":632.0,"rankHi":683.0},{"stateId":"phi-3-small-8k-instruct::default","configSlug":"phi-3-small-8k-instruct","modelId":"phi-3-small-8k-instruct","modelSlug":"phi-3-small-8k-instruct","name":"Phi-3-Small-8K-Instruct","modelName":"Phi-3-Small-8K-Instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":113.74,"scoreLo":108.87,"scoreHi":117.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":577.4756,"rankLo":539.975,"rankHi":620.0},{"stateId":"phi-4::default","configSlug":"phi-4","modelId":"phi-4","modelSlug":"phi-4","name":"Phi-4","modelName":"Phi-4","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":44,"score":121.73,"scoreLo":118.38,"scoreHi":124.66,"reasoningOffset":0.0,"rankEligible":true,"rankMean":479.1803,"rankLo":451.0,"rankHi":509.0},{"stateId":"phi-4-mini-instruct::default","configSlug":"phi-4-mini-instruct","modelId":"phi-4-mini-instruct","modelSlug":"phi-4-mini-instruct","name":"Phi-4-mini-instruct","modelName":"Phi-4-mini-instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":40,"score":107.0,"scoreLo":101.71,"scoreHi":111.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":656.7906,"rankLo":630.0,"rankHi":682.0},{"stateId":"phi-4-multimodal-instruct::default","configSlug":"phi-4-multimodal-instruct","modelId":"phi-4-multimodal-instruct","modelSlug":"phi-4-multimodal-instruct","name":"Phi-4-multimodal-instruct","modelName":"Phi-4-multimodal-instruct","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":110.33,"scoreLo":104.78,"scoreHi":115.05,"reasoningOffset":0.0,"rankEligible":true,"rankMean":619.1744,"rankLo":577.975,"rankHi":659.0},{"stateId":"phi-4-reasoning-plus::default","configSlug":"phi-4-reasoning-plus","modelId":"phi-4-reasoning-plus","modelSlug":"phi-4-reasoning-plus","name":"Phi-4-reasoning-plus","modelName":"Phi-4-reasoning-plus","organizationSlug":"microsoft","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":134.18,"scoreLo":129.47,"scoreHi":138.95,"reasoningOffset":2.4,"rankEligible":true,"rankMean":319.1222,"rankLo":258.0,"rankHi":377.0},{"stateId":"pixtral-12b-2409::default","configSlug":"pixtral-12b-2409","modelId":"pixtral-12b-2409","modelSlug":"pixtral-12b-2409","name":"Pixtral 12B","modelName":"Pixtral 12B","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":112.8,"scoreLo":107.69,"scoreHi":117.16,"reasoningOffset":0.0,"rankEligible":true,"rankMean":588.9794,"rankLo":549.0,"rankHi":629.0},{"stateId":"pixtral-large-instruct-2411::default","configSlug":"pixtral-large-instruct-2411","modelId":"pixtral-large-instruct-2411","modelSlug":"pixtral-large-instruct-2411","name":"Pixtral Large","modelName":"Pixtral Large","organizationSlug":"mistral-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":126.66,"scoreLo":122.73,"scoreHi":130.37,"reasoningOffset":0.0,"rankEligible":true,"rankMean":415.8103,"rankLo":375.0,"rankHi":458.0},{"stateId":"qed-nano::default","configSlug":"qed-nano-matharena","modelId":"qed-nano","modelSlug":"qed-nano","name":"QED-Nano","modelName":"QED-Nano","organizationSlug":"lm-provers","reasoningSupport":"always_on","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":136.14,"scoreLo":133.4,"scoreHi":138.94,"reasoningOffset":2.4,"rankEligible":true,"rankMean":292.3933,"rankLo":258.0,"rankHi":326.0},{"stateId":"quasar-438b::effort:max","configSlug":"quasar-438b-max","modelId":"quasar-438b","modelSlug":"quasar-438b","name":"Quasar 438B (max)","modelName":"Quasar 438B","organizationSlug":"multiverse-computing","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":10,"score":153.5,"scoreLo":150.19,"scoreHi":157.35,"reasoningOffset":0.86,"rankEligible":true,"rankMean":97.6397,"rankLo":80.0,"rankHi":118.0},{"stateId":"quyen-pro-v0-1::default","configSlug":"quyen-pro-v0-1","modelId":"quyen-pro-v0-1","modelSlug":"quyen-pro-v0-1","name":"Quyen Pro V0.1","modelName":"Quyen Pro V0.1","organizationSlug":"vilm","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":104.47,"scoreLo":97.99,"scoreHi":110.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":680.7097,"rankLo":637.0,"rankHi":718.0},{"stateId":"qvq-72b-preview::default","configSlug":"qvq-72b-preview","modelId":"qvq-72b-preview","modelSlug":"qvq-72b-preview","name":"QVQ 72B Preview","modelName":"QVQ 72B Preview","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":126.4,"scoreLo":121.17,"scoreHi":131.47,"reasoningOffset":2.4,"rankEligible":true,"rankMean":419.1322,"rankLo":358.0,"rankHi":479.0},{"stateId":"qwen-1-8b::default","configSlug":"qwen-1-8b","modelId":"qwen-1-8b","modelSlug":"qwen-1-8b","name":"Qwen 1.8B","modelName":"Qwen 1.8B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":81.04,"scoreLo":68.98,"scoreHi":91.35,"reasoningOffset":0.0,"rankEligible":true,"rankMean":828.6122,"rankLo":797.0,"rankHi":856.0},{"stateId":"qwen-14b::default","configSlug":"qwen-14b","modelId":"qwen-14b","modelSlug":"qwen-14b","name":"Qwen 14B","modelName":"Qwen 14B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":100.79,"scoreLo":94.19,"scoreHi":106.36,"reasoningOffset":0.0,"rankEligible":true,"rankMean":712.2217,"rankLo":685.0,"rankHi":739.0},{"stateId":"qwen-14b-chat::default","configSlug":"qwen-14b-chat","modelId":"qwen-14b-chat","modelSlug":"qwen-14b-chat","name":"Qwen 14B Chat","modelName":"Qwen 14B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":100.24,"scoreLo":93.25,"scoreHi":106.33,"reasoningOffset":0.0,"rankEligible":true,"rankMean":716.6028,"rankLo":684.0,"rankHi":751.0},{"stateId":"qwen2-5-72b-instruct::default","configSlug":"qwen-2-5-72b-instruct","modelId":"qwen2-5-72b-instruct","modelSlug":"qwen2-5-72b-instruct","name":"Qwen2.5 72B Instruct","modelName":"Qwen2.5 72B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":61,"score":125.5,"scoreLo":122.85,"scoreHi":127.89,"reasoningOffset":0.0,"rankEligible":true,"rankMean":429.1925,"rankLo":409.0,"rankHi":451.0},{"stateId":"qwen3-5-27b::default","configSlug":"qwen-3-5-27b","modelId":"qwen3-5-27b","modelSlug":"qwen3-5-27b","name":"Qwen3.5-27B","modelName":"Qwen3.5-27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":87,"score":149.01,"scoreLo":146.81,"scoreHi":151.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":142.4844,"rankLo":131.0,"rankHi":154.025},{"stateId":"qwen3-5-27b::enabled","configSlug":"qwen-3-5-27b-thinking","modelId":"qwen3-5-27b","modelSlug":"qwen3-5-27b","name":"Qwen3.5-27B (thinking)","modelName":"Qwen3.5-27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":18,"score":148.58,"scoreLo":146.36,"scoreHi":151.21,"reasoningOffset":-0.42,"rankEligible":true},{"stateId":"qwen3-5-35b-a3b::default","configSlug":"qwen-3-5-35b-a3b","modelId":"qwen3-5-35b-a3b","modelSlug":"qwen3-5-35b-a3b","name":"Qwen3.5 35B A3B","modelName":"Qwen3.5 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":85,"score":147.02,"scoreLo":144.92,"scoreHi":149.46,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-6-35b-a3b::default","configSlug":"qwen-3-6-35b-a3b","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B","modelName":"Qwen3.6 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":67,"score":147.71,"scoreLo":145.54,"scoreHi":150.13,"reasoningOffset":0.0,"rankEligible":true,"rankMean":157.0481,"rankLo":144.0,"rankHi":170.0},{"stateId":"qwen-72b::default","configSlug":"qwen-72b","modelId":"qwen-72b","modelSlug":"qwen-72b","name":"Qwen 72B","modelName":"Qwen 72B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":110.79,"scoreLo":105.22,"scoreHi":115.44,"reasoningOffset":0.0,"rankEligible":true,"rankMean":613.6486,"rankLo":568.0,"rankHi":655.0},{"stateId":"qwen-72b-chat::default","configSlug":"qwen-72b-chat","modelId":"qwen-72b-chat","modelSlug":"qwen-72b-chat","name":"Qwen 72B Chat","modelName":"Qwen 72B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":114.33,"scoreLo":108.13,"scoreHi":119.8,"reasoningOffset":0.0,"rankEligible":true,"rankMean":570.3572,"rankLo":514.0,"rankHi":628.0},{"stateId":"qwen-7b::default","configSlug":"qwen-7b","modelId":"qwen-7b","modelSlug":"qwen-7b","name":"Qwen 7B","modelName":"Qwen 7B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":92.74,"scoreLo":84.97,"scoreHi":99.08,"reasoningOffset":0.0,"rankEligible":true,"rankMean":773.5608,"rankLo":749.0,"rankHi":797.0},{"stateId":"qwen-7b-chat::default","configSlug":"qwen-7b-chat","modelId":"qwen-7b-chat","modelSlug":"qwen-7b-chat","name":"Qwen 7B Chat","modelName":"Qwen 7B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":93.29,"scoreLo":84.03,"scoreHi":101.05,"reasoningOffset":0.0,"rankEligible":true,"rankMean":768.8822,"rankLo":725.0,"rankHi":806.0},{"stateId":"qwen-max-2025-01-25::default","configSlug":"qwen-max-2025-01-25","modelId":"qwen-max-2025-01-25","modelSlug":"qwen-max-2025-01-25","name":"Qwen2.5-Max","modelName":"Qwen2.5-Max","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":130.23,"scoreLo":126.44,"scoreHi":133.74,"reasoningOffset":0.0,"rankEligible":true,"rankMean":371.6394,"rankLo":330.0,"rankHi":412.0},{"stateId":"qwen-turbo-2024-11-01::default","configSlug":"qwen-turbo-2024-11-01","modelId":"qwen-turbo-2024-11-01","modelSlug":"qwen-turbo-2024-11-01","name":"Qwen2.5-Turbo","modelName":"Qwen2.5-Turbo","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":121.72,"scoreLo":117.62,"scoreHi":125.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":479.2347,"rankLo":435.0,"rankHi":520.0},{"stateId":"qwen-vl-chat::default","configSlug":"qwen-vl-chat","modelId":"qwen-vl-chat","modelSlug":"qwen-vl-chat","name":"Qwen-VL-Chat","modelName":"Qwen-VL-Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":91.0,"scoreLo":79.84,"scoreHi":100.47,"reasoningOffset":0.0,"rankEligible":true,"rankMean":782.0369,"rankLo":727.0,"rankHi":825.0},{"stateId":"qwen-vl-max-0809::default","configSlug":"qwen-vl-max-0809","modelId":"qwen-vl-max-0809","modelSlug":"qwen-vl-max-0809","name":"Qwen VL Max 0809","modelName":"Qwen VL Max 0809","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":129.9,"scoreLo":121.83,"scoreHi":138.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":374.5872,"rankLo":265.0,"rankHi":474.0},{"stateId":"qwen1-5-0-5b::default","configSlug":"qwen1-5-0-5b","modelId":"qwen1-5-0-5b","modelSlug":"qwen1-5-0-5b","name":"Qwen1.5 0.5B","modelName":"Qwen1.5 0.5B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":65.8,"scoreLo":52.41,"scoreHi":76.7,"reasoningOffset":0.0,"rankEligible":true,"rankMean":865.76,"rankLo":856.0,"rankHi":874.0},{"stateId":"qwen1-5-0-5b-chat::default","configSlug":"qwen1-5-0-5b-chat","modelId":"qwen1-5-0-5b-chat","modelSlug":"qwen1-5-0-5b-chat","name":"Qwen1.5 0.5B Chat","modelName":"Qwen1.5 0.5B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":59.21,"scoreLo":43.96,"scoreHi":71.77,"reasoningOffset":0.0,"rankEligible":true,"rankMean":872.8247,"rankLo":864.0,"rankHi":880.0},{"stateId":"qwen1-5-1-8b::default","configSlug":"qwen1-5-1-8b","modelId":"qwen1-5-1-8b","modelSlug":"qwen1-5-1-8b","name":"Qwen1.5 1.8B","modelName":"Qwen1.5 1.8B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":78.01,"scoreLo":67.22,"scoreHi":86.85,"reasoningOffset":0.0,"rankEligible":true,"rankMean":839.7611,"rankLo":823.0,"rankHi":855.025},{"stateId":"qwen1-5-1-8b-chat::default","configSlug":"qwen1-5-1-8b-chat","modelId":"qwen1-5-1-8b-chat","modelSlug":"qwen1-5-1-8b-chat","name":"Qwen1.5 1.8B Chat","modelName":"Qwen1.5 1.8B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":78.17,"scoreLo":67.87,"scoreHi":86.73,"reasoningOffset":0.0,"rankEligible":true,"rankMean":839.3442,"rankLo":824.0,"rankHi":854.0},{"stateId":"qwen1-5-110b::default","configSlug":"qwen1-5-110b","modelId":"qwen1-5-110b","modelSlug":"qwen1-5-110b","name":"Qwen1.5 110B","modelName":"Qwen1.5 110B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":115.69,"scoreLo":110.92,"scoreHi":120.08,"reasoningOffset":0.0,"rankEligible":true,"rankMean":553.8453,"rankLo":516.0,"rankHi":594.025},{"stateId":"qwen1-5-110b-chat::default","configSlug":"qwen1-5-110b-chat","modelId":"qwen1-5-110b-chat","modelSlug":"qwen1-5-110b-chat","name":"Qwen1.5 110B Chat","modelName":"Qwen1.5 110B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":31,"score":115.26,"scoreLo":111.04,"scoreHi":118.86,"reasoningOffset":0.0,"rankEligible":true,"rankMean":558.5564,"rankLo":533.0,"rankHi":586.0},{"stateId":"qwen1-5-14b::default","configSlug":"qwen1-5-14b","modelId":"qwen1-5-14b","modelSlug":"qwen1-5-14b","name":"Qwen1.5 14B","modelName":"Qwen1.5 14B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":103.78,"scoreLo":97.87,"scoreHi":108.9,"reasoningOffset":0.0,"rankEligible":true,"rankMean":687.4189,"rankLo":660.0,"rankHi":713.0},{"stateId":"qwen1-5-14b-chat::default","configSlug":"qwen1-5-14b-chat","modelId":"qwen1-5-14b-chat","modelSlug":"qwen1-5-14b-chat","name":"Qwen1.5 14B Chat","modelName":"Qwen1.5 14B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":104.62,"scoreLo":98.87,"scoreHi":109.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":679.9928,"rankLo":654.0,"rankHi":705.0},{"stateId":"qwen1-5-32b::default","configSlug":"qwen1-5-32b","modelId":"qwen1-5-32b","modelSlug":"qwen1-5-32b","name":"Qwen1.5 32B","modelName":"Qwen1.5 32B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":110.78,"scoreLo":105.92,"scoreHi":114.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":614.1111,"rankLo":583.0,"rankHi":644.0},{"stateId":"qwen1-5-32b-chat::default","configSlug":"qwen1-5-32b-chat","modelId":"qwen1-5-32b-chat","modelSlug":"qwen1-5-32b-chat","name":"Qwen1.5 32B Chat","modelName":"Qwen1.5 32B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":110.63,"scoreLo":105.99,"scoreHi":114.7,"reasoningOffset":0.0,"rankEligible":true,"rankMean":615.9058,"rankLo":588.0,"rankHi":643.0},{"stateId":"qwen1-5-4b::default","configSlug":"qwen1-5-4b","modelId":"qwen1-5-4b","modelSlug":"qwen1-5-4b","name":"Qwen1.5 4B","modelName":"Qwen1.5 4B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":89.49,"scoreLo":80.83,"scoreHi":96.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":792.6903,"rankLo":768.975,"rankHi":813.025},{"stateId":"qwen1-5-4b-chat::default","configSlug":"qwen1-5-4b-chat","modelId":"qwen1-5-4b-chat","modelSlug":"qwen1-5-4b-chat","name":"Qwen1.5 4B Chat","modelName":"Qwen1.5 4B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":83.86,"scoreLo":74.44,"scoreHi":91.74,"reasoningOffset":0.0,"rankEligible":true,"rankMean":818.6289,"rankLo":800.0,"rankHi":837.0},{"stateId":"qwen1-5-72b::default","configSlug":"qwen1-5-72b","modelId":"qwen1-5-72b","modelSlug":"qwen1-5-72b","name":"Qwen1.5 72B","modelName":"Qwen1.5 72B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":30,"score":110.46,"scoreLo":105.63,"scoreHi":114.73,"reasoningOffset":0.0,"rankEligible":true,"rankMean":618.005,"rankLo":584.0,"rankHi":648.0},{"stateId":"qwen1-5-72b-chat::default","configSlug":"qwen1-5-72b-chat","modelId":"qwen1-5-72b-chat","modelSlug":"qwen1-5-72b-chat","name":"Qwen1.5 72B Chat","modelName":"Qwen1.5 72B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":32,"score":112.38,"scoreLo":107.94,"scoreHi":116.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":594.5047,"rankLo":567.0,"rankHi":621.0},{"stateId":"qwen1-5-7b::default","configSlug":"qwen1-5-7b","modelId":"qwen1-5-7b","modelSlug":"qwen1-5-7b","name":"Qwen1.5 7B","modelName":"Qwen1.5 7B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":33,"score":96.72,"scoreLo":90.36,"scoreHi":102.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":744.9397,"rankLo":730.0,"rankHi":759.0},{"stateId":"qwen1-5-7b-chat::default","configSlug":"qwen1-5-7b-chat","modelId":"qwen1-5-7b-chat","modelSlug":"qwen1-5-7b-chat","name":"Qwen1.5 7B Chat","modelName":"Qwen1.5 7B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":35,"score":96.15,"scoreLo":89.61,"scoreHi":101.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":749.9325,"rankLo":736.0,"rankHi":764.0},{"stateId":"qwen1-5-moe-a2-7b::default","configSlug":"qwen1-5-moe-a2-7b","modelId":"qwen1-5-moe-a2-7b","modelSlug":"qwen1-5-moe-a2-7b","name":"Qwen1.5 MoE A2.7B","modelName":"Qwen1.5 MoE A2.7B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":95.56,"scoreLo":88.13,"scoreHi":101.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":753.7561,"rankLo":723.0,"rankHi":781.0},{"stateId":"qwen1-5-moe-a2-7b-chat::default","configSlug":"qwen1-5-moe-a2-7b-chat","modelId":"qwen1-5-moe-a2-7b-chat","modelSlug":"qwen1-5-moe-a2-7b-chat","name":"Qwen1.5 MoE A2.7B Chat","modelName":"Qwen1.5 MoE A2.7B Chat","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":95.52,"scoreLo":88.11,"scoreHi":101.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":754.2133,"rankLo":727.975,"rankHi":777.0},{"stateId":"qwen2-0-5b::default","configSlug":"qwen2-0-5b","modelId":"qwen2-0-5b","modelSlug":"qwen2-0-5b","name":"Qwen2-0.5B","modelName":"Qwen2-0.5B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":72.56,"scoreLo":61.39,"scoreHi":82.22,"reasoningOffset":0.0,"rankEligible":true,"rankMean":854.8264,"rankLo":841.975,"rankHi":865.0},{"stateId":"qwen2-0-5b-instruct::default","configSlug":"qwen2-0-5b-instruct","modelId":"qwen2-0-5b-instruct","modelSlug":"qwen2-0-5b-instruct","name":"Qwen2 0.5B Instruct","modelName":"Qwen2 0.5B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":63.33,"scoreLo":48.47,"scoreHi":75.68,"reasoningOffset":0.0,"rankEligible":true,"rankMean":868.4878,"rankLo":858.0,"rankHi":878.0},{"stateId":"qwen2-1-5b::default","configSlug":"qwen2-1-5b","modelId":"qwen2-1-5b","modelSlug":"qwen2-1-5b","name":"Qwen2-1.5B","modelName":"Qwen2-1.5B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":88.82,"scoreLo":80.59,"scoreHi":95.79,"reasoningOffset":0.0,"rankEligible":true,"rankMean":796.4411,"rankLo":775.0,"rankHi":815.0},{"stateId":"qwen2-1-5b-instruct::default","configSlug":"qwen2-1-5b-instruct","modelId":"qwen2-1-5b-instruct","modelSlug":"qwen2-1-5b-instruct","name":"Qwen2 1.5B Instruct","modelName":"Qwen2 1.5B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":89.36,"scoreLo":80.79,"scoreHi":96.71,"reasoningOffset":0.0,"rankEligible":true,"rankMean":793.2806,"rankLo":771.0,"rankHi":814.0},{"stateId":"qwen2-5-0-5b::default","configSlug":"qwen2-5-0-5b","modelId":"qwen2-5-0-5b","modelSlug":"qwen2-5-0-5b","name":"Qwen2.5 0.5B","modelName":"Qwen2.5 0.5B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":78.13,"scoreLo":67.54,"scoreHi":86.87,"reasoningOffset":0.0,"rankEligible":true,"rankMean":839.3892,"rankLo":823.0,"rankHi":855.0},{"stateId":"qwen2-5-0-5b-instruct::default","configSlug":"qwen2-5-0-5b-instruct","modelId":"qwen2-5-0-5b-instruct","modelSlug":"qwen2-5-0-5b-instruct","name":"Qwen2.5 0.5B Instruct","modelName":"Qwen2.5 0.5B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":75.54,"scoreLo":63.86,"scoreHi":85.45,"reasoningOffset":0.0,"rankEligible":true,"rankMean":846.8308,"rankLo":827.0,"rankHi":863.0},{"stateId":"qwen2-5-1-5b::default","configSlug":"qwen2-5-1-5b","modelId":"qwen2-5-1-5b","modelSlug":"qwen2-5-1-5b","name":"Qwen2.5 1.5B","modelName":"Qwen2.5 1.5B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":95.35,"scoreLo":88.02,"scoreHi":101.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":755.3969,"rankLo":727.0,"rankHi":781.0},{"stateId":"qwen2-5-1-5b-instruct::default","configSlug":"qwen2-5-1-5b-instruct","modelId":"qwen2-5-1-5b-instruct","modelSlug":"qwen2-5-1-5b-instruct","name":"Qwen2.5 1.5B Instruct","modelName":"Qwen2.5 1.5B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":97.86,"scoreLo":90.18,"scoreHi":104.36,"reasoningOffset":0.0,"rankEligible":true,"rankMean":735.5264,"rankLo":700.0,"rankHi":770.0},{"stateId":"qwen2-5-14b::default","configSlug":"qwen2-5-14b","modelId":"qwen2-5-14b","modelSlug":"qwen2-5-14b","name":"Qwen2.5 14B","modelName":"Qwen2.5 14B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":116.06,"scoreLo":111.25,"scoreHi":120.38,"reasoningOffset":0.0,"rankEligible":true,"rankMean":549.5039,"rankLo":510.0,"rankHi":591.0},{"stateId":"qwen2-5-14b-instruct::default","configSlug":"qwen2-5-14b-instruct","modelId":"qwen2-5-14b-instruct","modelSlug":"qwen2-5-14b-instruct","name":"Qwen2.5 14B Instruct","modelName":"Qwen2.5 14B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":31,"score":120.54,"scoreLo":117.02,"scoreHi":123.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":495.8608,"rankLo":466.0,"rankHi":523.0},{"stateId":"qwen2-5-14b-instruct-1m::default","configSlug":"qwen2-5-14b-instruct-1m","modelId":"qwen2-5-14b-instruct-1m","modelSlug":"qwen2-5-14b-instruct-1m","name":"Qwen2.5 14B Instruct 1M","modelName":"Qwen2.5 14B Instruct 1M","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":123.41,"scoreLo":118.9,"scoreHi":127.84,"reasoningOffset":0.0,"rankEligible":true,"rankMean":456.6053,"rankLo":406.0,"rankHi":509.0},{"stateId":"qwen2-5-32b::default","configSlug":"qwen2-5-32b","modelId":"qwen2-5-32b","modelSlug":"qwen2-5-32b","name":"Qwen2.5 32B","modelName":"Qwen2.5 32B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":122.28,"scoreLo":118.87,"scoreHi":125.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":471.4397,"rankLo":441.0,"rankHi":504.0},{"stateId":"qwen2-5-32b-instruct::default","configSlug":"qwen2-5-32b-instruct","modelId":"qwen2-5-32b-instruct","modelSlug":"qwen2-5-32b-instruct","name":"Qwen2.5 Instruct 32B","modelName":"Qwen2.5 Instruct 32B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":38,"score":123.95,"scoreLo":121.18,"scoreHi":126.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":448.8733,"rankLo":425.0,"rankHi":473.0},{"stateId":"qwen2-5-3b::default","configSlug":"qwen2-5-3b","modelId":"qwen2-5-3b","modelSlug":"qwen2-5-3b","name":"Qwen2.5 3B","modelName":"Qwen2.5 3B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":100.89,"scoreLo":94.52,"scoreHi":106.55,"reasoningOffset":0.0,"rankEligible":true,"rankMean":711.4572,"rankLo":683.0,"rankHi":739.0},{"stateId":"qwen2-5-3b-instruct::default","configSlug":"qwen2-5-3b-instruct","modelId":"qwen2-5-3b-instruct","modelSlug":"qwen2-5-3b-instruct","name":"Qwen2.5 3B Instruct","modelName":"Qwen2.5 3B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":38,"score":104.73,"scoreLo":99.56,"scoreHi":109.49,"reasoningOffset":0.0,"rankEligible":true,"rankMean":679.2053,"rankLo":658.0,"rankHi":699.0},{"stateId":"qwen2-5-72b::default","configSlug":"qwen2-5-72b","modelId":"qwen2-5-72b","modelSlug":"qwen2-5-72b","name":"Qwen2.5 72B","modelName":"Qwen2.5 72B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":39,"score":121.75,"scoreLo":118.24,"scoreHi":124.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":478.7617,"rankLo":445.0,"rankHi":511.0},{"stateId":"qwen2-5-7b::default","configSlug":"qwen2-5-7b","modelId":"qwen2-5-7b","modelSlug":"qwen2-5-7b","name":"Qwen2.5 7B","modelName":"Qwen2.5 7B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":19,"score":110.67,"scoreLo":105.37,"scoreHi":115.18,"reasoningOffset":0.0,"rankEligible":true,"rankMean":615.2436,"rankLo":577.0,"rankHi":652.0},{"stateId":"qwen2-5-7b-instruct::default","configSlug":"qwen2-5-7b-instruct","modelId":"qwen2-5-7b-instruct","modelSlug":"qwen2-5-7b-instruct","name":"Qwen2.5 7B Instruct","modelName":"Qwen2.5 7B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":52,"score":113.82,"scoreLo":109.79,"scoreHi":117.29,"reasoningOffset":0.0,"rankEligible":true,"rankMean":576.1453,"rankLo":553.0,"rankHi":600.0},{"stateId":"qwen2-5-7b-instruct-1m::default","configSlug":"qwen2-5-7b-instruct-1m","modelId":"qwen2-5-7b-instruct-1m","modelSlug":"qwen2-5-7b-instruct-1m","name":"Qwen2.5 7B Instruct 1M","modelName":"Qwen2.5 7B Instruct 1M","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":115.74,"scoreLo":110.75,"scoreHi":120.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":553.3728,"rankLo":512.0,"rankHi":597.0},{"stateId":"qwen2-5-coder-0-5b::default","configSlug":"qwen2-5-coder-0-5b","modelId":"qwen2-5-coder-0-5b","modelSlug":"qwen2-5-coder-0-5b","name":"Qwen2.5 Coder 0.5B","modelName":"Qwen2.5 Coder 0.5B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":75.36,"scoreLo":63.85,"scoreHi":84.92,"reasoningOffset":0.0,"rankEligible":true,"rankMean":847.5117,"rankLo":831.0,"rankHi":862.0},{"stateId":"qwen2-5-coder-0-5b-instruct::default","configSlug":"qwen2-5-coder-0-5b-instruct","modelId":"qwen2-5-coder-0-5b-instruct","modelSlug":"qwen2-5-coder-0-5b-instruct","name":"Qwen2.5 Coder 0.5B Instruct","modelName":"Qwen2.5 Coder 0.5B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":101.43,"scoreLo":89.38,"scoreHi":112.52,"reasoningOffset":0.0,"rankEligible":true,"rankMean":703.9411,"rankLo":594.0,"rankHi":784.0},{"stateId":"qwen2-5-coder-1-5b::default","configSlug":"qwen2-5-coder-1-5b","modelId":"qwen2-5-coder-1-5b","modelSlug":"qwen2-5-coder-1-5b","name":"Qwen2.5 Coder 1.5B","modelName":"Qwen2.5 Coder 1.5B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":91.59,"scoreLo":83.52,"scoreHi":98.41,"reasoningOffset":0.0,"rankEligible":true,"rankMean":780.5756,"rankLo":753.0,"rankHi":805.0},{"stateId":"qwen2-5-coder-1-5b-instruct::default","configSlug":"qwen2-5-coder-1-5b-instruct","modelId":"qwen2-5-coder-1-5b-instruct","modelSlug":"qwen2-5-coder-1-5b-instruct","name":"Qwen2.5 Coder 1.5B Instruct","modelName":"Qwen2.5 Coder 1.5B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":109.8,"scoreLo":99.36,"scoreHi":120.08,"reasoningOffset":0.0,"rankEligible":true,"rankMean":622.3794,"rankLo":508.0,"rankHi":716.0},{"stateId":"qwen2-5-coder-14b::default","configSlug":"qwen2-5-coder-14b","modelId":"qwen2-5-coder-14b","modelSlug":"qwen2-5-coder-14b","name":"Qwen2.5-Coder-14B","modelName":"Qwen2.5-Coder-14B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":113.02,"scoreLo":108.33,"scoreHi":117.35,"reasoningOffset":0.0,"rankEligible":true,"rankMean":586.3144,"rankLo":549.0,"rankHi":622.0},{"stateId":"qwen2-5-coder-14b-instruct::default","configSlug":"qwen2-5-coder-14b-instruct","modelId":"qwen2-5-coder-14b-instruct","modelSlug":"qwen2-5-coder-14b-instruct","name":"Qwen2.5 Coder 14B Instruct","modelName":"Qwen2.5 Coder 14B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":119.17,"scoreLo":113.83,"scoreHi":124.15,"reasoningOffset":0.0,"rankEligible":true,"rankMean":512.1519,"rankLo":452.0,"rankHi":564.025},{"stateId":"qwen2-5-coder-32b::default","configSlug":"qwen2-5-coder-32b","modelId":"qwen2-5-coder-32b","modelSlug":"qwen2-5-coder-32b","name":"Qwen2.5-Coder-32B","modelName":"Qwen2.5-Coder-32B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":117.28,"scoreLo":112.27,"scoreHi":121.81,"reasoningOffset":0.0,"rankEligible":true,"rankMean":535.3689,"rankLo":487.0,"rankHi":580.0},{"stateId":"qwen2-5-coder-32b-instruct::default","configSlug":"qwen2-5-coder-32b-instruct","modelId":"qwen2-5-coder-32b-instruct","modelSlug":"qwen2-5-coder-32b-instruct","name":"Qwen2.5 Coder 32B Instruct","modelName":"Qwen2.5 Coder 32B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":121.03,"scoreLo":117.3,"scoreHi":124.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":488.7428,"rankLo":449.0,"rankHi":526.0},{"stateId":"qwen2-5-coder-3b::default","configSlug":"qwen2-5-coder-3b","modelId":"qwen2-5-coder-3b","modelSlug":"qwen2-5-coder-3b","name":"Qwen2.5-Coder-3B","modelName":"Qwen2.5-Coder-3B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":14,"score":100.23,"scoreLo":93.25,"scoreHi":106.23,"reasoningOffset":0.0,"rankEligible":true,"rankMean":716.6114,"rankLo":685.0,"rankHi":750.0},{"stateId":"qwen2-5-coder-3b-instruct::default","configSlug":"qwen2-5-coder-3b-instruct","modelId":"qwen2-5-coder-3b-instruct","modelSlug":"qwen2-5-coder-3b-instruct","name":"Qwen2.5 Coder 3B Instruct","modelName":"Qwen2.5 Coder 3B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":104.85,"scoreLo":97.99,"scoreHi":111.04,"reasoningOffset":0.0,"rankEligible":true,"rankMean":676.9089,"rankLo":631.0,"rankHi":717.0},{"stateId":"qwen2-5-coder-7b::default","configSlug":"qwen2-5-coder-7b","modelId":"qwen2-5-coder-7b","modelSlug":"qwen2-5-coder-7b","name":"Qwen2.5 Coder 7B","modelName":"Qwen2.5 Coder 7B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":105.86,"scoreLo":100.1,"scoreHi":110.74,"reasoningOffset":0.0,"rankEligible":true,"rankMean":668.1769,"rankLo":637.0,"rankHi":696.0},{"stateId":"qwen2-5-coder-7b-instruct::default","configSlug":"qwen2-5-coder-7b-instruct","modelId":"qwen2-5-coder-7b-instruct","modelSlug":"qwen2-5-coder-7b-instruct","name":"Qwen2.5-Coder-7B-Instruct","modelName":"Qwen2.5-Coder-7B-Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":110.47,"scoreLo":105.07,"scoreHi":115.25,"reasoningOffset":0.0,"rankEligible":true,"rankMean":617.5469,"rankLo":575.0,"rankHi":658.0},{"stateId":"qwen2-5-vl-32b-instruct::default","configSlug":"qwen2-5-vl-32b-instruct","modelId":"qwen2-5-vl-32b-instruct","modelSlug":"qwen2-5-vl-32b-instruct","name":"Qwen2.5 VL 32B Instruct","modelName":"Qwen2.5 VL 32B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":127.09,"scoreLo":123.86,"scoreHi":129.89,"reasoningOffset":0.0,"rankEligible":true,"rankMean":410.7164,"rankLo":382.0,"rankHi":440.0},{"stateId":"qwen2-5-vl-3b-instruct::default","configSlug":"qwen2-5-vl-3b-instruct","modelId":"qwen2-5-vl-3b-instruct","modelSlug":"qwen2-5-vl-3b-instruct","name":"Qwen2.5-VL-3B-Instruct","modelName":"Qwen2.5-VL-3B-Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":29,"score":113.41,"scoreLo":108.5,"scoreHi":117.65,"reasoningOffset":0.0,"rankEligible":true,"rankMean":581.5494,"rankLo":545.0,"rankHi":620.0},{"stateId":"qwen2-5-vl-72b-instruct::default","configSlug":"qwen2-5-vl-72b-instruct","modelId":"qwen2-5-vl-72b-instruct","modelSlug":"qwen2-5-vl-72b-instruct","name":"Qwen2.5-VL-72B-Instruct","modelName":"Qwen2.5-VL-72B-Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":51,"score":131.0,"scoreLo":128.41,"scoreHi":133.5,"reasoningOffset":0.0,"rankEligible":true,"rankMean":361.4172,"rankLo":335.0,"rankHi":389.0},{"stateId":"qwen2-5-vl-7b-instruct::default","configSlug":"qwen2-5-vl-7b-instruct","modelId":"qwen2-5-vl-7b-instruct","modelSlug":"qwen2-5-vl-7b-instruct","name":"Qwen2.5-VL-7B-Instruct","modelName":"Qwen2.5-VL-7B-Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":35,"score":120.63,"scoreLo":116.93,"scoreHi":124.03,"reasoningOffset":0.0,"rankEligible":true,"rankMean":494.3619,"rankLo":460.0,"rankHi":525.0},{"stateId":"qwen2-57b-a14b::default","configSlug":"qwen2-57b-a14b","modelId":"qwen2-57b-a14b","modelSlug":"qwen2-57b-a14b","name":"Qwen2-57B-A14B","modelName":"Qwen2-57B-A14B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":111.09,"scoreLo":105.79,"scoreHi":115.88,"reasoningOffset":0.0,"rankEligible":true,"rankMean":610.1242,"rankLo":567.0,"rankHi":650.0},{"stateId":"qwen2-57b-a14b-instruct::default","configSlug":"qwen2-57b-a14b-instruct","modelId":"qwen2-57b-a14b-instruct","modelSlug":"qwen2-57b-a14b-instruct","name":"Qwen2-57B-A14B Instruct","modelName":"Qwen2-57B-A14B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":112.06,"scoreLo":106.93,"scoreHi":116.46,"reasoningOffset":0.0,"rankEligible":true,"rankMean":598.3247,"rankLo":561.0,"rankHi":634.0},{"stateId":"qwen2-72b::default","configSlug":"qwen2-72b","modelId":"qwen2-72b","modelSlug":"qwen2-72b","name":"Qwen2-72B","modelName":"Qwen2-72B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":121.67,"scoreLo":118.04,"scoreHi":124.91,"reasoningOffset":0.0,"rankEligible":true,"rankMean":479.9922,"rankLo":445.0,"rankHi":515.0},{"stateId":"qwen2-72b-instruct::default","configSlug":"qwen2-72b-instruct","modelId":"qwen2-72b-instruct","modelSlug":"qwen2-72b-instruct","name":"Qwen2 72B Instruct","modelName":"Qwen2 72B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":35,"score":121.06,"scoreLo":117.84,"scoreHi":123.85,"reasoningOffset":0.0,"rankEligible":true,"rankMean":488.7644,"rankLo":462.0,"rankHi":516.0},{"stateId":"qwen2-7b::default","configSlug":"qwen2-7b","modelId":"qwen2-7b","modelSlug":"qwen2-7b","name":"Qwen2-7B","modelName":"Qwen2-7B","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":106.46,"scoreLo":100.99,"scoreHi":111.18,"reasoningOffset":0.0,"rankEligible":true,"rankMean":662.2106,"rankLo":630.0,"rankHi":689.0},{"stateId":"qwen2-7b-instruct::default","configSlug":"qwen2-7b-instruct","modelId":"qwen2-7b-instruct","modelSlug":"qwen2-7b-instruct","name":"Qwen2 7B Instruct","modelName":"Qwen2 7B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":20,"score":108.26,"scoreLo":103.31,"scoreHi":112.56,"reasoningOffset":0.0,"rankEligible":true,"rankMean":643.1381,"rankLo":614.975,"rankHi":669.0},{"stateId":"qwen2-vl-2b-instruct::default","configSlug":"qwen2-vl-2b-instruct","modelId":"qwen2-vl-2b-instruct","modelSlug":"qwen2-vl-2b-instruct","name":"Qwen2 VL 2B Instruct","modelName":"Qwen2 VL 2B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":104.15,"scoreLo":97.78,"scoreHi":109.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":683.8653,"rankLo":648.975,"rankHi":716.0},{"stateId":"qwen2-vl-72b-instruct::default","configSlug":"qwen2-vl-72b-instruct","modelId":"qwen2-vl-72b-instruct","modelSlug":"qwen2-vl-72b-instruct","name":"Qwen2 VL 72B Instruct","modelName":"Qwen2 VL 72B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":34,"score":125.71,"scoreLo":122.6,"scoreHi":128.51,"reasoningOffset":0.0,"rankEligible":true,"rankMean":426.7072,"rankLo":401.0,"rankHi":455.0},{"stateId":"qwen2-vl-7b-instruct::default","configSlug":"qwen2-vl-7b-instruct","modelId":"qwen2-vl-7b-instruct","modelSlug":"qwen2-vl-7b-instruct","name":"Qwen2 VL 7B Instruct","modelName":"Qwen2 VL 7B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":33,"score":112.52,"scoreLo":107.82,"scoreHi":116.43,"reasoningOffset":0.0,"rankEligible":true,"rankMean":592.7378,"rankLo":562.0,"rankHi":623.0},{"stateId":"qwen3-0-6b::default","configSlug":"qwen3-0-6b","modelId":"qwen3-0-6b","modelSlug":"qwen3-0-6b","name":"Qwen3 0.6B","modelName":"Qwen3 0.6B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":93.3,"scoreLo":85.32,"scoreHi":99.96,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-0-6b-base::default","configSlug":"qwen3-0-6b-base","modelId":"qwen3-0-6b-base","modelSlug":"qwen3-0-6b-base","name":"Qwen3 0.6B Base","modelName":"Qwen3 0.6B Base","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":90.62,"scoreLo":82.46,"scoreHi":97.44,"reasoningOffset":0.0,"rankEligible":true,"rankMean":786.3358,"rankLo":764.0,"rankHi":808.0},{"stateId":"qwen3-0-6b::disabled","configSlug":"qwen3-0-6b-no-thinking","modelId":"qwen3-0-6b","modelSlug":"qwen3-0-6b","name":"Qwen3 0.6B (disabled)","modelName":"Qwen3 0.6B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":23,"score":91.7,"scoreLo":84.06,"scoreHi":98.12,"reasoningOffset":-1.59,"rankEligible":true},{"stateId":"qwen3-0-6b::enabled","configSlug":"qwen3-0-6b-thinking","modelId":"qwen3-0-6b","modelSlug":"qwen3-0-6b","name":"Qwen3 0.6B (thinking)","modelName":"Qwen3 0.6B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":21,"score":97.08,"scoreLo":90.11,"scoreHi":102.93,"reasoningOffset":3.79,"rankEligible":true,"rankMean":741.8908,"rankLo":715.0,"rankHi":769.0},{"stateId":"qwen3-1-7b::default","configSlug":"qwen3-1-7b","modelId":"qwen3-1-7b","modelSlug":"qwen3-1-7b","name":"Qwen3 1.7B","modelName":"Qwen3 1.7B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":109.51,"scoreLo":104.63,"scoreHi":113.97,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-1-7b-base::default","configSlug":"qwen3-1-7b-base","modelId":"qwen3-1-7b-base","modelSlug":"qwen3-1-7b-base","name":"Qwen3 1.7B Base","modelName":"Qwen3 1.7B Base","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":101.87,"scoreLo":95.46,"scoreHi":107.34,"reasoningOffset":0.0,"rankEligible":true,"rankMean":703.5806,"rankLo":675.0,"rankHi":730.0},{"stateId":"qwen3-1-7b::disabled","configSlug":"qwen3-1-7b-no-thinking","modelId":"qwen3-1-7b","modelSlug":"qwen3-1-7b","name":"Qwen3 1.7B (disabled)","modelName":"Qwen3 1.7B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":28,"score":107.14,"scoreLo":102.13,"scoreHi":111.51,"reasoningOffset":-2.36,"rankEligible":true},{"stateId":"qwen3-1-7b::enabled","configSlug":"qwen3-1-7b-thinking","modelId":"qwen3-1-7b","modelSlug":"qwen3-1-7b","name":"Qwen3 1.7B (thinking)","modelName":"Qwen3 1.7B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":31,"score":113.45,"scoreLo":109.33,"scoreHi":117.1,"reasoningOffset":3.94,"rankEligible":true,"rankMean":580.8914,"rankLo":555.0,"rankHi":607.0},{"stateId":"qwen3-14b::default","configSlug":"qwen3-14b","modelId":"qwen3-14b","modelSlug":"qwen3-14b","name":"Qwen3 14B","modelName":"Qwen3 14B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":131.99,"scoreLo":129.23,"scoreHi":134.52,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-14b-base::default","configSlug":"qwen3-14b-base","modelId":"qwen3-14b-base","modelSlug":"qwen3-14b-base","name":"Qwen3 14B Base","modelName":"Qwen3 14B Base","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":120.65,"scoreLo":116.49,"scoreHi":124.39,"reasoningOffset":0.0,"rankEligible":true,"rankMean":493.8247,"rankLo":451.0,"rankHi":533.0},{"stateId":"qwen3-14b::disabled","configSlug":"qwen3-14b-no-thinking","modelId":"qwen3-14b","modelSlug":"qwen3-14b","name":"Qwen3 14B (disabled)","modelName":"Qwen3 14B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":24,"score":128.63,"scoreLo":125.95,"scoreHi":131.08,"reasoningOffset":-3.36,"rankEligible":true},{"stateId":"qwen3-14b::enabled","configSlug":"qwen3-14b-thinking","modelId":"qwen3-14b","modelSlug":"qwen3-14b","name":"Qwen3 14B (thinking)","modelName":"Qwen3 14B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":32,"score":133.44,"scoreLo":131.26,"scoreHi":135.47,"reasoningOffset":1.45,"rankEligible":true,"rankMean":329.7292,"rankLo":306.0,"rankHi":352.0},{"stateId":"qwen3-235b-a22b::default","configSlug":"qwen3-235b-a22b","modelId":"qwen3-235b-a22b","modelSlug":"qwen3-235b-a22b","name":"Qwen3-235B-A22B","modelName":"Qwen3-235B-A22B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":60,"score":139.03,"scoreLo":137.26,"scoreHi":140.89,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-235b-a22b-instruct-2507::default","configSlug":"qwen3-235b-a22b-2507","modelId":"qwen3-235b-a22b-instruct-2507","modelSlug":"qwen3-235b-a22b-instruct-2507","name":"Qwen3 235B A22B Instruct 2507","modelName":"Qwen3 235B A22B Instruct 2507","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":54,"score":140.64,"scoreLo":138.72,"scoreHi":142.61,"reasoningOffset":0.0,"rankEligible":true,"rankMean":231.4183,"rankLo":216.0,"rankHi":250.0},{"stateId":"qwen3-235b-a22b-thinking-2507::default","configSlug":"qwen3-235b-a22b-2507-thinking","modelId":"qwen3-235b-a22b-thinking-2507","modelSlug":"qwen3-235b-a22b-thinking-2507","name":"Qwen3 235B A22B Thinking 2507","modelName":"Qwen3 235B A22B Thinking 2507","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":58,"score":142.64,"scoreLo":140.6,"scoreHi":144.74,"reasoningOffset":2.4,"rankEligible":true,"rankMean":210.0811,"rankLo":197.0,"rankHi":223.0},{"stateId":"qwen3-235b-a22b::disabled","configSlug":"qwen3-235b-a22b-no-thinking","modelId":"qwen3-235b-a22b","modelSlug":"qwen3-235b-a22b","name":"Qwen3-235B-A22B (disabled)","modelName":"Qwen3-235B-A22B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":33,"score":135.0,"scoreLo":132.98,"scoreHi":136.95,"reasoningOffset":-4.03,"rankEligible":true},{"stateId":"qwen3-235b-a22b::enabled","configSlug":"qwen3-235b-a22b-thinking","modelId":"qwen3-235b-a22b","modelSlug":"qwen3-235b-a22b","name":"Qwen3-235B-A22B (thinking)","modelName":"Qwen3-235B-A22B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":36,"score":139.73,"scoreLo":138.04,"scoreHi":141.48,"reasoningOffset":0.7,"rankEligible":true,"rankMean":243.4011,"rankLo":228.0,"rankHi":259.0},{"stateId":"qwen3-30b-a3b::default","configSlug":"qwen3-30b-a3b","modelId":"qwen3-30b-a3b","modelSlug":"qwen3-30b-a3b","name":"Qwen3-30B-A3B","modelName":"Qwen3-30B-A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":21,"score":131.88,"scoreLo":129.64,"scoreHi":134.02,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-30b-a3b-instruct-2507::default","configSlug":"qwen3-30b-a3b-2507","modelId":"qwen3-30b-a3b-instruct-2507","modelSlug":"qwen3-30b-a3b-instruct-2507","name":"Qwen3 30B A3B Instruct 2507","modelName":"Qwen3 30B A3B Instruct 2507","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":38,"score":134.42,"scoreLo":132.28,"scoreHi":136.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":316.1967,"rankLo":292.0,"rankHi":340.0},{"stateId":"qwen3-30b-a3b-thinking-2507::default","configSlug":"qwen3-30b-a3b-2507-thinking","modelId":"qwen3-30b-a3b-thinking-2507","modelSlug":"qwen3-30b-a3b-thinking-2507","name":"Qwen3 30B A3B Thinking 2507","modelName":"Qwen3 30B A3B Thinking 2507","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":62,"score":136.86,"scoreLo":135.09,"scoreHi":138.61,"reasoningOffset":2.4,"rankEligible":true,"rankMean":282.7572,"rankLo":265.0,"rankHi":301.0},{"stateId":"qwen3-30b-a3b-base::default","configSlug":"qwen3-30b-a3b-base","modelId":"qwen3-30b-a3b-base","modelSlug":"qwen3-30b-a3b-base","name":"Qwen3-30B-A3B-Base","modelName":"Qwen3-30B-A3B-Base","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":121.48,"scoreLo":117.54,"scoreHi":125.11,"reasoningOffset":0.0,"rankEligible":true,"rankMean":482.6314,"rankLo":441.0,"rankHi":522.0},{"stateId":"qwen3-30b-a3b::disabled","configSlug":"qwen3-30b-a3b-no-thinking","modelId":"qwen3-30b-a3b","modelSlug":"qwen3-30b-a3b","name":"Qwen3-30B-A3B (disabled)","modelName":"Qwen3-30B-A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":31,"score":128.64,"scoreLo":126.35,"scoreHi":130.77,"reasoningOffset":-3.24,"rankEligible":true},{"stateId":"qwen3-30b-a3b::enabled","configSlug":"qwen3-30b-a3b-thinking","modelId":"qwen3-30b-a3b","modelSlug":"qwen3-30b-a3b","name":"Qwen3-30B-A3B (thinking)","modelName":"Qwen3-30B-A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":31,"score":133.49,"scoreLo":131.57,"scoreHi":135.35,"reasoningOffset":1.61,"rankEligible":true,"rankMean":329.2158,"rankLo":309.0,"rankHi":347.025},{"stateId":"qwen3-32b::default","configSlug":"qwen3-32b","modelId":"qwen3-32b","modelSlug":"qwen3-32b","name":"Qwen3 32B","modelName":"Qwen3 32B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":20,"score":133.31,"scoreLo":131.16,"scoreHi":135.49,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-32b::effort:4000","configSlug":"qwen3-32b-4000","modelId":"qwen3-32b","modelSlug":"qwen3-32b","name":"Qwen3 32B (4000)","modelName":"Qwen3 32B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"4000","reasoningCondition":"effort:4000","isDefault":false,"n":1,"score":131.92,"scoreLo":127.04,"scoreHi":136.65,"reasoningOffset":-1.39,"rankEligible":false},{"stateId":"qwen3-32b::disabled","configSlug":"qwen3-32b-no-thinking","modelId":"qwen3-32b","modelSlug":"qwen3-32b","name":"Qwen3 32B (disabled)","modelName":"Qwen3 32B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":27,"score":129.82,"scoreLo":127.48,"scoreHi":131.94,"reasoningOffset":-3.49,"rankEligible":true},{"stateId":"qwen3-32b::enabled","configSlug":"qwen3-32b-thinking","modelId":"qwen3-32b","modelSlug":"qwen3-32b","name":"Qwen3 32B (thinking)","modelName":"Qwen3 32B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":37,"score":134.96,"scoreLo":133.06,"scoreHi":136.79,"reasoningOffset":1.64,"rankEligible":true,"rankMean":308.5908,"rankLo":289.0,"rankHi":329.0},{"stateId":"qwen3-4b::default","configSlug":"qwen3-4b","modelId":"qwen3-4b","modelSlug":"qwen3-4b","name":"Qwen3 4B","modelName":"Qwen3 4B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":121.73,"scoreLo":118.09,"scoreHi":124.82,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-4b-instruct-2507::default","configSlug":"qwen3-4b-2507","modelId":"qwen3-4b-instruct-2507","modelSlug":"qwen3-4b-instruct-2507","name":"Qwen3 4B Instruct 2507","modelName":"Qwen3 4B Instruct 2507","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":30,"score":126.87,"scoreLo":124.07,"scoreHi":129.35,"reasoningOffset":0.0,"rankEligible":true,"rankMean":413.1597,"rankLo":391.0,"rankHi":437.0},{"stateId":"qwen3-4b-thinking-2507::default","configSlug":"qwen3-4b-2507-thinking","modelId":"qwen3-4b-thinking-2507","modelSlug":"qwen3-4b-thinking-2507","name":"Qwen3 4B Thinking 2507","modelName":"Qwen3 4B Thinking 2507","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":41,"score":132.28,"scoreLo":130.07,"scoreHi":134.36,"reasoningOffset":2.4,"rankEligible":true,"rankMean":344.9158,"rankLo":323.0,"rankHi":366.0},{"stateId":"qwen3-4b-base::default","configSlug":"qwen3-4b-base","modelId":"qwen3-4b-base","modelSlug":"qwen3-4b-base","name":"Qwen3 4B Base","modelName":"Qwen3 4B Base","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":113.14,"scoreLo":108.15,"scoreHi":117.41,"reasoningOffset":0.0,"rankEligible":true,"rankMean":584.9278,"rankLo":546.0,"rankHi":625.0},{"stateId":"qwen3-4b::disabled","configSlug":"qwen3-4b-no-thinking","modelId":"qwen3-4b","modelSlug":"qwen3-4b","name":"Qwen3 4B (disabled)","modelName":"Qwen3 4B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":26,"score":119.75,"scoreLo":116.39,"scoreHi":122.76,"reasoningOffset":-1.97,"rankEligible":true},{"stateId":"qwen3-4b::enabled","configSlug":"qwen3-4b-thinking","modelId":"qwen3-4b","modelSlug":"qwen3-4b","name":"Qwen3 4B (thinking)","modelName":"Qwen3 4B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":25,"score":125.17,"scoreLo":122.46,"scoreHi":127.73,"reasoningOffset":3.45,"rankEligible":true,"rankMean":433.315,"rankLo":412.0,"rankHi":457.0},{"stateId":"qwen3-5-0-8b::default","configSlug":"qwen3-5-0-8b","modelId":"qwen3-5-0-8b","modelSlug":"qwen3-5-0-8b","name":"Qwen3.5-0.8B","modelName":"Qwen3.5-0.8B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":3,"score":99.49,"scoreLo":92.81,"scoreHi":104.95,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"qwen3-5-0-8b::disabled","configSlug":"qwen3-5-0-8b-no-thinking","modelId":"qwen3-5-0-8b","modelSlug":"qwen3-5-0-8b","name":"Qwen3.5-0.8B (disabled)","modelName":"Qwen3.5-0.8B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":41,"score":101.72,"scoreLo":95.68,"scoreHi":106.64,"reasoningOffset":2.23,"rankEligible":true},{"stateId":"qwen3-5-0-8b::enabled","configSlug":"qwen3-5-0-8b-thinking","modelId":"qwen3-5-0-8b","modelSlug":"qwen3-5-0-8b","name":"Qwen3.5-0.8B (thinking)","modelName":"Qwen3.5-0.8B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":54,"score":103.47,"scoreLo":97.75,"scoreHi":108.23,"reasoningOffset":3.98,"rankEligible":true,"rankMean":690.3808,"rankLo":669.0,"rankHi":710.0},{"stateId":"qwen3-5-122b-a10b::default","configSlug":"qwen3-5-122b-a10b","modelId":"qwen3-5-122b-a10b","modelSlug":"qwen3-5-122b-a10b","name":"Qwen3.5 122B A10B","modelName":"Qwen3.5 122B A10B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":79,"score":149.11,"scoreLo":146.86,"scoreHi":151.72,"reasoningOffset":0.0,"rankEligible":true,"rankMean":141.4,"rankLo":130.0,"rankHi":153.0},{"stateId":"qwen3-5-122b-a10b::effort:high","configSlug":"qwen3-5-122b-a10b-high","modelId":"qwen3-5-122b-a10b","modelSlug":"qwen3-5-122b-a10b","name":"Qwen3.5 122B A10B (high)","modelName":"Qwen3.5 122B A10B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":149.27,"scoreLo":146.07,"scoreHi":152.91,"reasoningOffset":0.17,"rankEligible":false},{"stateId":"qwen3-5-122b-a10b::disabled","configSlug":"qwen3-5-122b-a10b-no-thinking","modelId":"qwen3-5-122b-a10b","modelSlug":"qwen3-5-122b-a10b","name":"Qwen3.5 122B A10B (disabled)","modelName":"Qwen3.5 122B A10B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":12,"score":146.98,"scoreLo":144.6,"scoreHi":149.76,"reasoningOffset":-2.13,"rankEligible":true},{"stateId":"qwen3-5-122b-a10b::enabled","configSlug":"qwen3-5-122b-a10b-thinking","modelId":"qwen3-5-122b-a10b","modelSlug":"qwen3-5-122b-a10b","name":"Qwen3.5 122B A10B (thinking)","modelName":"Qwen3.5 122B A10B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":20,"score":148.86,"scoreLo":146.52,"scoreHi":151.62,"reasoningOffset":-0.25,"rankEligible":true},{"stateId":"qwen3-5-27b::effort:high","configSlug":"qwen3-5-27b-high","modelId":"qwen3-5-27b","modelSlug":"qwen3-5-27b","name":"Qwen3.5-27B (high)","modelName":"Qwen3.5-27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":149.18,"scoreLo":146.02,"scoreHi":152.77,"reasoningOffset":0.17,"rankEligible":false},{"stateId":"qwen3-5-27b::effort:low","configSlug":"qwen3-5-27b-low","modelId":"qwen3-5-27b","modelSlug":"qwen3-5-27b","name":"Qwen3.5-27B (low)","modelName":"Qwen3.5-27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":146.47,"scoreLo":142.31,"scoreHi":150.99,"reasoningOffset":-2.54,"rankEligible":false},{"stateId":"qwen3-5-27b::disabled","configSlug":"qwen3-5-27b-no-thinking","modelId":"qwen3-5-27b","modelSlug":"qwen3-5-27b","name":"Qwen3.5-27B (disabled)","modelName":"Qwen3.5-27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":145.48,"scoreLo":142.37,"scoreHi":148.82,"reasoningOffset":-3.53,"rankEligible":true},{"stateId":"qwen3-5-2b::default","configSlug":"qwen3-5-2b","modelId":"qwen3-5-2b","modelSlug":"qwen3-5-2b","name":"Qwen3.5-2B","modelName":"Qwen3.5-2B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":2,"score":118.04,"scoreLo":113.9,"scoreHi":121.77,"reasoningOffset":0.0,"rankEligible":false},{"stateId":"qwen3-5-2b::disabled","configSlug":"qwen3-5-2b-no-thinking","modelId":"qwen3-5-2b","modelSlug":"qwen3-5-2b","name":"Qwen3.5-2B (disabled)","modelName":"Qwen3.5-2B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":38,"score":119.94,"scoreLo":116.7,"scoreHi":122.99,"reasoningOffset":1.9,"rankEligible":true},{"stateId":"qwen3-5-2b::enabled","configSlug":"qwen3-5-2b-thinking","modelId":"qwen3-5-2b","modelSlug":"qwen3-5-2b","name":"Qwen3.5-2B (thinking)","modelName":"Qwen3.5-2B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":74,"score":121.96,"scoreLo":118.92,"scoreHi":124.77,"reasoningOffset":3.91,"rankEligible":true,"rankMean":475.7458,"rankLo":450.0,"rankHi":502.0},{"stateId":"qwen3-5-35b-a3b::effort:16k","configSlug":"qwen3-5-35b-a3b-16k","modelId":"qwen3-5-35b-a3b","modelSlug":"qwen3-5-35b-a3b","name":"Qwen3.5 35B A3B (16k)","modelName":"Qwen3.5 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":1,"score":147.28,"scoreLo":144.7,"scoreHi":150.06,"reasoningOffset":0.26,"rankEligible":false},{"stateId":"qwen3-5-35b-a3b::effort:high","configSlug":"qwen3-5-35b-a3b-high","modelId":"qwen3-5-35b-a3b","modelSlug":"qwen3-5-35b-a3b","name":"Qwen3.5 35B A3B (high)","modelName":"Qwen3.5 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":147.05,"scoreLo":143.84,"scoreHi":150.55,"reasoningOffset":0.03,"rankEligible":false},{"stateId":"qwen3-5-35b-a3b::effort:minimal","configSlug":"qwen3-5-35b-a3b-minimal","modelId":"qwen3-5-35b-a3b","modelSlug":"qwen3-5-35b-a3b","name":"Qwen3.5 35B A3B (minimal)","modelName":"Qwen3.5 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":142.58,"scoreLo":137.87,"scoreHi":147.41,"reasoningOffset":-4.44,"rankEligible":false},{"stateId":"qwen3-5-35b-a3b::disabled","configSlug":"qwen3-5-35b-a3b-no-thinking","modelId":"qwen3-5-35b-a3b","modelSlug":"qwen3-5-35b-a3b","name":"Qwen3.5 35B A3B (disabled)","modelName":"Qwen3.5 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":13,"score":144.38,"scoreLo":141.99,"scoreHi":146.97,"reasoningOffset":-2.64,"rankEligible":true},{"stateId":"qwen3-5-35b-a3b::enabled","configSlug":"qwen3-5-35b-a3b-thinking","modelId":"qwen3-5-35b-a3b","modelSlug":"qwen3-5-35b-a3b","name":"Qwen3.5 35B A3B (thinking)","modelName":"Qwen3.5 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":18,"score":147.23,"scoreLo":145.06,"scoreHi":149.69,"reasoningOffset":0.21,"rankEligible":true,"rankMean":162.2286,"rankLo":148.0,"rankHi":176.0},{"stateId":"qwen3-5-397b-a17b::default","configSlug":"qwen3-5-397b-a17b","modelId":"qwen3-5-397b-a17b","modelSlug":"qwen3-5-397b-a17b","name":"Qwen3.5 397B A17B","modelName":"Qwen3.5 397B A17B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":111,"score":150.57,"scoreLo":148.28,"scoreHi":153.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":125.3886,"rankLo":116.0,"rankHi":135.0},{"stateId":"qwen3-5-397b-a17b::effort:high","configSlug":"qwen3-5-397b-a17b-high","modelId":"qwen3-5-397b-a17b","modelSlug":"qwen3-5-397b-a17b","name":"Qwen3.5 397B A17B (high)","modelName":"Qwen3.5 397B A17B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":150.21,"scoreLo":147.32,"scoreHi":153.47,"reasoningOffset":-0.36,"rankEligible":false},{"stateId":"qwen3-5-397b-a17b::disabled","configSlug":"qwen3-5-397b-a17b-no-thinking","modelId":"qwen3-5-397b-a17b","modelSlug":"qwen3-5-397b-a17b","name":"Qwen3.5 397B A17B (disabled)","modelName":"Qwen3.5 397B A17B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":17,"score":147.55,"scoreLo":145.06,"scoreHi":150.36,"reasoningOffset":-3.02,"rankEligible":true},{"stateId":"qwen3-5-397b-a17b::enabled","configSlug":"qwen3-5-397b-a17b-thinking","modelId":"qwen3-5-397b-a17b","modelSlug":"qwen3-5-397b-a17b","name":"Qwen3.5 397B A17B (thinking)","modelName":"Qwen3.5 397B A17B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":29,"score":150.4,"scoreLo":148.0,"scoreHi":153.18,"reasoningOffset":-0.17,"rankEligible":true},{"stateId":"qwen3-5-397b-a17b::effort:xhigh","configSlug":"qwen3-5-397b-a17b-xhigh","modelId":"qwen3-5-397b-a17b","modelSlug":"qwen3-5-397b-a17b","name":"Qwen3.5 397B A17B (xhigh)","modelName":"Qwen3.5 397B A17B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":151.13,"scoreLo":147.86,"scoreHi":154.8,"reasoningOffset":0.56,"rankEligible":false},{"stateId":"qwen3-5-4b::disabled","configSlug":"qwen3-5-4b-no-thinking","modelId":"qwen3-5-4b","modelSlug":"qwen3-5-4b","name":"Qwen3.5-4B (disabled)","modelName":"Qwen3.5-4B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":9,"score":136.85,"scoreLo":134.05,"scoreHi":139.65,"reasoningOffset":-2.72,"rankEligible":true},{"stateId":"qwen3-5-4b::enabled","configSlug":"qwen3-5-4b-thinking","modelId":"qwen3-5-4b","modelSlug":"qwen3-5-4b","name":"Qwen3.5-4B (thinking)","modelName":"Qwen3.5-4B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":58,"score":140.98,"scoreLo":139.19,"scoreHi":142.96,"reasoningOffset":1.41,"rankEligible":true,"rankMean":226.9161,"rankLo":214.0,"rankHi":244.0},{"stateId":"qwen3-5-4b::default","configSlug":"qwen3-5-4b-unknown","modelId":"qwen3-5-4b","modelSlug":"qwen3-5-4b","name":"Qwen3.5-4B","modelName":"Qwen3.5-4B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":66,"score":139.57,"scoreLo":137.71,"scoreHi":141.56,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-5-9b::default","configSlug":"qwen3-5-9b","modelId":"qwen3-5-9b","modelSlug":"qwen3-5-9b","name":"Qwen3.5-9B","modelName":"Qwen3.5-9B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":76,"score":142.21,"scoreLo":140.34,"scoreHi":144.21,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-5-9b::effort:16k","configSlug":"qwen3-5-9b-16k","modelId":"qwen3-5-9b","modelSlug":"qwen3-5-9b","name":"Qwen3.5-9B (16k)","modelName":"Qwen3.5-9B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":1,"score":142.44,"scoreLo":140.1,"scoreHi":145.04,"reasoningOffset":0.24,"rankEligible":false},{"stateId":"qwen3-5-9b::disabled","configSlug":"qwen3-5-9b-no-thinking","modelId":"qwen3-5-9b","modelSlug":"qwen3-5-9b","name":"Qwen3.5-9B (disabled)","modelName":"Qwen3.5-9B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":11,"score":141.11,"scoreLo":138.8,"scoreHi":143.58,"reasoningOffset":-1.1,"rankEligible":true},{"stateId":"qwen3-5-9b::enabled","configSlug":"qwen3-5-9b-thinking","modelId":"qwen3-5-9b","modelSlug":"qwen3-5-9b","name":"Qwen3.5-9B (thinking)","modelName":"Qwen3.5-9B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":21,"score":142.66,"scoreLo":140.79,"scoreHi":144.78,"reasoningOffset":0.45,"rankEligible":true,"rankMean":209.9106,"rankLo":198.0,"rankHi":222.0},{"stateId":"qwen3-5-omni-flash::default","configSlug":"qwen3-5-omni-flash","modelId":"qwen3-5-omni-flash","modelSlug":"qwen3-5-omni-flash","name":"Qwen3.5-Omni-Flash","modelName":"Qwen3.5-Omni-Flash","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":41,"score":138.64,"scoreLo":136.52,"scoreHi":140.78,"reasoningOffset":0.0,"rankEligible":true,"rankMean":258.3786,"rankLo":234.975,"rankHi":282.0},{"stateId":"qwen3-5-omni-plus::default","configSlug":"qwen3-5-omni-plus","modelId":"qwen3-5-omni-plus","modelSlug":"qwen3-5-omni-plus","name":"Qwen3.5-Omni-Plus","modelName":"Qwen3.5-Omni-Plus","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":42,"score":146.46,"scoreLo":144.0,"scoreHi":149.12,"reasoningOffset":0.0,"rankEligible":true,"rankMean":170.4789,"rankLo":151.975,"rankHi":189.0},{"stateId":"qwen3-5-plus::default","configSlug":"qwen3-5-plus","modelId":"qwen3-5-plus","modelSlug":"qwen3-5-plus","name":"Qwen3.5 Plus (2026-02-15)","modelName":"Qwen3.5 Plus (2026-02-15)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":14,"score":148.62,"scoreLo":146.16,"scoreHi":151.49,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-5-plus::effort:high","configSlug":"qwen3-5-plus-high","modelId":"qwen3-5-plus","modelSlug":"qwen3-5-plus","name":"Qwen3.5 Plus (2026-02-15) (high)","modelName":"Qwen3.5 Plus (2026-02-15)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":149.34,"scoreLo":145.9,"scoreHi":153.12,"reasoningOffset":0.72,"rankEligible":false},{"stateId":"qwen3-5-plus::disabled","configSlug":"qwen3-5-plus-no-thinking","modelId":"qwen3-5-plus","modelSlug":"qwen3-5-plus","name":"Qwen3.5 Plus (2026-02-15) (disabled)","modelName":"Qwen3.5 Plus (2026-02-15)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":39,"score":148.28,"scoreLo":145.91,"scoreHi":150.91,"reasoningOffset":-0.34,"rankEligible":true},{"stateId":"qwen3-5-plus::enabled","configSlug":"qwen3-5-plus-thinking","modelId":"qwen3-5-plus","modelSlug":"qwen3-5-plus","name":"Qwen3.5 Plus (2026-02-15) (thinking)","modelName":"Qwen3.5 Plus (2026-02-15)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":20,"score":149.13,"scoreLo":146.49,"scoreHi":152.19,"reasoningOffset":0.51,"rankEligible":true,"rankMean":141.1725,"rankLo":120.0,"rankHi":162.0},{"stateId":"qwen3-6-27b::default","configSlug":"qwen3-6-27b","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B","modelName":"Qwen3.6 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":61,"score":149.64,"scoreLo":147.31,"scoreHi":152.5,"reasoningOffset":0.0,"rankEligible":true,"rankMean":135.7072,"rankLo":123.0,"rankHi":148.0},{"stateId":"qwen3-6-27b::effort:high","configSlug":"qwen3-6-27b-high","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B (high)","modelName":"Qwen3.6 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":3,"score":149.21,"scoreLo":146.23,"scoreHi":152.59,"reasoningOffset":-0.43,"rankEligible":false},{"stateId":"qwen3-6-27b::effort:low","configSlug":"qwen3-6-27b-low","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B (low)","modelName":"Qwen3.6 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":147.06,"scoreLo":142.73,"scoreHi":151.68,"reasoningOffset":-2.58,"rankEligible":false},{"stateId":"qwen3-6-27b::effort:medium","configSlug":"qwen3-6-27b-medium","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B (medium)","modelName":"Qwen3.6 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":148.97,"scoreLo":144.76,"scoreHi":153.66,"reasoningOffset":-0.67,"rankEligible":false},{"stateId":"qwen3-6-27b::disabled","configSlug":"qwen3-6-27b-no-thinking","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B (disabled)","modelName":"Qwen3.6 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":16,"score":146.98,"scoreLo":144.49,"scoreHi":149.81,"reasoningOffset":-2.66,"rankEligible":true},{"stateId":"qwen3-6-27b::enabled","configSlug":"qwen3-6-27b-thinking","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B (thinking)","modelName":"Qwen3.6 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":22,"score":149.49,"scoreLo":146.85,"scoreHi":152.39,"reasoningOffset":-0.15,"rankEligible":true},{"stateId":"qwen3-6-27b::effort:xhigh","configSlug":"qwen3-6-27b-xhigh","modelId":"qwen3-6-27b","modelSlug":"qwen3-6-27b","name":"Qwen3.6 27B (xhigh)","modelName":"Qwen3.6 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":150.3,"scoreLo":146.99,"scoreHi":154.16,"reasoningOffset":0.66,"rankEligible":false},{"stateId":"qwen3-6-35b-a3b::effort:high","configSlug":"qwen3-6-35b-a3b-high","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B (high)","modelName":"Qwen3.6 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":147.68,"scoreLo":144.51,"scoreHi":151.24,"reasoningOffset":-0.03,"rankEligible":false},{"stateId":"qwen3-6-35b-a3b::effort:low","configSlug":"qwen3-6-35b-a3b-low","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B (low)","modelName":"Qwen3.6 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":2,"score":145.26,"scoreLo":141.41,"scoreHi":149.35,"reasoningOffset":-2.44,"rankEligible":false},{"stateId":"qwen3-6-35b-a3b::effort:medium","configSlug":"qwen3-6-35b-a3b-medium","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B (medium)","modelName":"Qwen3.6 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":147.16,"scoreLo":142.93,"scoreHi":151.65,"reasoningOffset":-0.55,"rankEligible":false},{"stateId":"qwen3-6-35b-a3b::disabled","configSlug":"qwen3-6-35b-a3b-no-thinking","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B (disabled)","modelName":"Qwen3.6 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":17,"score":145.31,"scoreLo":142.95,"scoreHi":147.88,"reasoningOffset":-2.4,"rankEligible":true},{"stateId":"qwen3-6-35b-a3b::enabled","configSlug":"qwen3-6-35b-a3b-thinking","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B (thinking)","modelName":"Qwen3.6 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":22,"score":146.88,"scoreLo":144.44,"scoreHi":149.46,"reasoningOffset":-0.83,"rankEligible":true},{"stateId":"qwen3-6-35b-a3b::effort:xhigh","configSlug":"qwen3-6-35b-a3b-xhigh","modelId":"qwen3-6-35b-a3b","modelSlug":"qwen3-6-35b-a3b","name":"Qwen3.6 35B A3B (xhigh)","modelName":"Qwen3.6 35B A3B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":148.38,"scoreLo":145.27,"scoreHi":151.83,"reasoningOffset":0.68,"rankEligible":false},{"stateId":"qwen3-6-max-preview::default","configSlug":"qwen3-6-max","modelId":"qwen3-6-max-preview","modelSlug":"qwen3-6-max-preview","name":"Qwen3.6-Max-Preview","modelName":"Qwen3.6-Max-Preview","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":153.17,"scoreLo":150.37,"scoreHi":156.49,"reasoningOffset":0.0,"rankEligible":true,"rankMean":100.3994,"rankLo":87.0,"rankHi":114.0},{"stateId":"qwen3-6-max-preview::disabled","configSlug":"qwen3-6-max-preview-no-thinking","modelId":"qwen3-6-max-preview","modelSlug":"qwen3-6-max-preview","name":"Qwen3.6-Max-Preview (disabled)","modelName":"Qwen3.6-Max-Preview","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":149.56,"scoreLo":143.2,"scoreHi":155.99,"reasoningOffset":-3.6,"rankEligible":false},{"stateId":"qwen3-6-max-preview::enabled","configSlug":"qwen3-6-max-preview-thinking","modelId":"qwen3-6-max-preview","modelSlug":"qwen3-6-max-preview","name":"Qwen3.6-Max-Preview (thinking)","modelName":"Qwen3.6-Max-Preview","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":3,"score":153.98,"scoreLo":150.24,"scoreHi":158.21,"reasoningOffset":0.81,"rankEligible":false},{"stateId":"qwen3-6-plus::default","configSlug":"qwen3-6-plus","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":102,"score":151.29,"scoreLo":148.96,"scoreHi":154.06,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-6-plus::effort:high","configSlug":"qwen3-6-plus-high","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02) (high)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":8,"score":150.6,"scoreLo":147.93,"scoreHi":153.68,"reasoningOffset":-0.69,"rankEligible":true},{"stateId":"qwen3-6-plus::effort:low","configSlug":"qwen3-6-plus-low","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02) (low)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":148.81,"scoreLo":144.55,"scoreHi":153.28,"reasoningOffset":-2.48,"rankEligible":false},{"stateId":"qwen3-6-plus::effort:medium","configSlug":"qwen3-6-plus-medium","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02) (medium)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":1,"score":150.63,"scoreLo":146.38,"scoreHi":155.21,"reasoningOffset":-0.67,"rankEligible":false},{"stateId":"qwen3-6-plus::effort:minimal","configSlug":"qwen3-6-plus-minimal","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02) (minimal)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":147.16,"scoreLo":142.51,"scoreHi":152.26,"reasoningOffset":-4.13,"rankEligible":false},{"stateId":"qwen3-6-plus::disabled","configSlug":"qwen3-6-plus-no-thinking","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02) (disabled)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":147.99,"scoreLo":144.48,"scoreHi":151.75,"reasoningOffset":-3.31,"rankEligible":true},{"stateId":"qwen3-6-plus::enabled","configSlug":"qwen3-6-plus-thinking","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02) (thinking)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":13,"score":151.69,"scoreLo":148.93,"scoreHi":154.98,"reasoningOffset":0.4,"rankEligible":true,"rankMean":114.0036,"rankLo":98.0,"rankHi":133.025},{"stateId":"qwen3-6-plus::effort:xhigh","configSlug":"qwen3-6-plus-xhigh","modelId":"qwen3-6-plus","modelSlug":"qwen3-6-plus","name":"Qwen3.6 Plus (2026-04-02) (xhigh)","modelName":"Qwen3.6 Plus (2026-04-02)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":152.03,"scoreLo":148.64,"scoreHi":155.73,"reasoningOffset":0.74,"rankEligible":false},{"stateId":"qwen3-7-flash::default","configSlug":"qwen3-7-flash","modelId":"qwen3-7-flash","modelSlug":"qwen3-7-flash","name":"Qwen3.7 Flash","modelName":"Qwen3.7 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":11,"score":146.76,"scoreLo":143.41,"scoreHi":150.33,"reasoningOffset":0.0,"rankEligible":true,"rankMean":167.2039,"rankLo":136.0,"rankHi":198.0},{"stateId":"qwen3-7-flash::disabled","configSlug":"qwen3-7-flash-no-thinking","modelId":"qwen3-7-flash","modelSlug":"qwen3-7-flash","name":"Qwen3.7 Flash (disabled)","modelName":"Qwen3.7 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":null,"reasoningCondition":"disabled","isDefault":false,"n":7,"score":144.38,"scoreLo":141.29,"scoreHi":147.62,"reasoningOffset":-2.37,"rankEligible":true},{"stateId":"qwen3-7-flash::enabled","configSlug":"qwen3-7-flash-thinking-context-arena-mrcr-v2","modelId":"qwen3-7-flash","modelSlug":"qwen3-7-flash","name":"Qwen3.7 Flash (thinking)","modelName":"Qwen3.7 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":2,"score":147.81,"scoreLo":143.51,"scoreHi":152.35,"reasoningOffset":1.05,"rankEligible":false},{"stateId":"qwen3-8-2-4t-a95b::default","configSlug":"qwen3-8-2-4t-a95b","modelId":"qwen3-8-2-4t-a95b","modelSlug":"qwen3-8-2-4t-a95b","name":"Qwen3.8 2.4T A95B","modelName":"Qwen3.8 2.4T A95B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":159.63,"scoreLo":156.1,"scoreHi":163.67,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-8-2-4t-a95b::enabled","configSlug":"qwen3-8-2-4t-a95b-wordlebench-high","modelId":"qwen3-8-2-4t-a95b","modelSlug":"qwen3-8-2-4t-a95b","name":"Qwen3.8 2.4T A95B (enabled)","modelName":"Qwen3.8 2.4T A95B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":160.44,"scoreLo":156.02,"scoreHi":165.49,"reasoningOffset":0.82,"rankEligible":false},{"stateId":"qwen3-8-2-4t-a95b::effort:xhigh","configSlug":"qwen3-8-2-4t-a95b-xhigh","modelId":"qwen3-8-2-4t-a95b","modelSlug":"qwen3-8-2-4t-a95b","name":"Qwen3.8 2.4T A95B (xhigh)","modelName":"Qwen3.8 2.4T A95B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":8,"score":159.84,"scoreLo":156.03,"scoreHi":164.4,"reasoningOffset":0.21,"rankEligible":true,"rankMean":50.5256,"rankLo":28.0,"rankHi":70.0},{"stateId":"qwen3-8-27b::default","configSlug":"qwen3-8-27b","modelId":"qwen3-8-27b","modelSlug":"qwen3-8-27b","name":"Qwen3.8 27B","modelName":"Qwen3.8 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":47,"score":155.82,"scoreLo":152.99,"scoreHi":159.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":80.0875,"rankLo":76.0,"rankHi":86.0},{"stateId":"qwen3-8-27b::effort:low","configSlug":"qwen3-8-27b-low","modelId":"qwen3-8-27b","modelSlug":"qwen3-8-27b","name":"Qwen3.8 27B (low)","modelName":"Qwen3.8 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":19,"score":153.64,"scoreLo":150.96,"scoreHi":156.83,"reasoningOffset":-2.18,"rankEligible":true},{"stateId":"qwen3-8-27b::effort:medium","configSlug":"qwen3-8-27b-medium","modelId":"qwen3-8-27b","modelSlug":"qwen3-8-27b","name":"Qwen3.8 27B (medium)","modelName":"Qwen3.8 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":20,"score":154.03,"scoreLo":151.31,"scoreHi":157.17,"reasoningOffset":-1.78,"rankEligible":true},{"stateId":"qwen3-8-27b::disabled","configSlug":"qwen3-8-27b-no-thinking","modelId":"qwen3-8-27b","modelSlug":"qwen3-8-27b","name":"Qwen3.8 27B (disabled)","modelName":"Qwen3.8 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":null,"reasoningCondition":"disabled","isDefault":false,"n":19,"score":148.42,"scoreLo":145.88,"scoreHi":151.13,"reasoningOffset":-7.39,"rankEligible":true},{"stateId":"qwen3-8-27b::enabled","configSlug":"qwen3-8-27b-thinking","modelId":"qwen3-8-27b","modelSlug":"qwen3-8-27b","name":"Qwen3.8 27B (thinking)","modelName":"Qwen3.8 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":2,"score":155.99,"scoreLo":152.08,"scoreHi":160.47,"reasoningOffset":0.18,"rankEligible":false},{"stateId":"qwen3-8-27b::effort:xhigh","configSlug":"qwen3-8-27b-xhigh","modelId":"qwen3-8-27b","modelSlug":"qwen3-8-27b","name":"Qwen3.8 27B (xhigh)","modelName":"Qwen3.8 27B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":51,"score":155.72,"scoreLo":152.91,"scoreHi":159.02,"reasoningOffset":-0.1,"rankEligible":true},{"stateId":"qwen3-8-flash::default","configSlug":"qwen3-8-flash","modelId":"qwen3-8-flash","modelSlug":"qwen3-8-flash","name":"Qwen3.8 Flash","modelName":"Qwen3.8 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":8,"score":154.95,"scoreLo":150.15,"scoreHi":159.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":87.5017,"rankLo":61.975,"rankHi":120.0},{"stateId":"qwen3-8-flash::effort:max","configSlug":"qwen3-8-flash-max","modelId":"qwen3-8-flash","modelSlug":"qwen3-8-flash","name":"Qwen3.8 Flash (max)","modelName":"Qwen3.8 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":1,"score":156.09,"scoreLo":150.74,"scoreHi":161.63,"reasoningOffset":1.14,"rankEligible":false},{"stateId":"qwen3-8-flash-next::default","configSlug":"qwen3-8-flash-next","modelId":"qwen3-8-flash-next","modelSlug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","modelName":"Qwen3.8-Flash-Next","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":24,"score":159.46,"scoreLo":156.02,"scoreHi":163.59,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-8-flash-next::disabled","configSlug":"qwen3-8-flash-next-no-thinking","modelId":"qwen3-8-flash-next","modelSlug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next (disabled)","modelName":"Qwen3.8-Flash-Next","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":153.85,"scoreLo":147.16,"scoreHi":160.95,"reasoningOffset":-5.61,"rankEligible":false},{"stateId":"qwen3-8-flash-next::enabled","configSlug":"qwen3-8-flash-next-thinking","modelId":"qwen3-8-flash-next","modelSlug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next (thinking)","modelName":"Qwen3.8-Flash-Next","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":13,"score":159.96,"scoreLo":156.39,"scoreHi":164.14,"reasoningOffset":0.5,"rankEligible":true,"rankMean":49.2528,"rankLo":32.0,"rankHi":66.0},{"stateId":"qwen3-8-flash-next::effort:xhigh","configSlug":"qwen3-8-flash-next-xhigh","modelId":"qwen3-8-flash-next","modelSlug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next (xhigh)","modelName":"Qwen3.8-Flash-Next","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":160.36,"scoreLo":156.13,"scoreHi":165.2,"reasoningOffset":0.89,"rankEligible":false},{"stateId":"qwen3-8-flash::disabled","configSlug":"qwen3-8-flash-no-thinking","modelId":"qwen3-8-flash","modelSlug":"qwen3-8-flash","name":"Qwen3.8 Flash (disabled)","modelName":"Qwen3.8 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":null,"reasoningCondition":"disabled","isDefault":false,"n":2,"score":147.27,"scoreLo":141.26,"scoreHi":153.91,"reasoningOffset":-7.68,"rankEligible":false},{"stateId":"qwen3-8-flash::enabled","configSlug":"qwen3-8-flash-wordlebench-high","modelId":"qwen3-8-flash","modelSlug":"qwen3-8-flash","name":"Qwen3.8 Flash (enabled)","modelName":"Qwen3.8 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":156.1,"scoreLo":150.74,"scoreHi":161.8,"reasoningOffset":1.15,"rankEligible":false},{"stateId":"qwen3-8-flash::effort:xhigh","configSlug":"qwen3-8-flash-xhigh-codex","modelId":"qwen3-8-flash","modelSlug":"qwen3-8-flash","name":"Qwen3.8 Flash (xhigh)","modelName":"Qwen3.8 Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":156.04,"scoreLo":150.78,"scoreHi":161.8,"reasoningOffset":1.09,"rankEligible":false},{"stateId":"qwen3-8-max::default","configSlug":"qwen3-8-max","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max","modelName":"Qwen3.8-Max","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":72,"score":160.13,"scoreLo":156.83,"scoreHi":164.11,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-8-max-0902::default","configSlug":"qwen3-8-max-0902","modelId":"qwen3-8-max-0902","modelSlug":"qwen3-8-max-0902","name":"Qwen3.8 Max (0902)","modelName":"Qwen3.8 Max (0902)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":161.44,"scoreLo":158.05,"scoreHi":165.64,"reasoningOffset":0.0,"rankEligible":true,"rankMean":33.5647,"rankLo":24.0,"rankHi":45.0},{"stateId":"qwen3-8-max-0902::effort:max","configSlug":"qwen3-8-max-0902-max-claude-code","modelId":"qwen3-8-max-0902","modelSlug":"qwen3-8-max-0902","name":"Qwen3.8 Max (0902) (max)","modelName":"Qwen3.8 Max (0902)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"max","reasoningCondition":"effort:max","isDefault":false,"n":1,"score":162.14,"scoreLo":158.25,"scoreHi":166.99,"reasoningOffset":0.7,"rankEligible":false},{"stateId":"qwen3-8-max-0902::enabled","configSlug":"qwen3-8-max-0902-thinking","modelId":"qwen3-8-max-0902","modelSlug":"qwen3-8-max-0902","name":"Qwen3.8 Max (0902) (thinking)","modelName":"Qwen3.8 Max (0902)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":20,"score":161.43,"scoreLo":157.93,"scoreHi":165.62,"reasoningOffset":-0.0,"rankEligible":true},{"stateId":"qwen3-8-max-0902::effort:xhigh","configSlug":"qwen3-8-max-0902-xhigh-claude-code","modelId":"qwen3-8-max-0902","modelSlug":"qwen3-8-max-0902","name":"Qwen3.8 Max (0902) (xhigh)","modelName":"Qwen3.8 Max (0902)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":162.28,"scoreLo":158.25,"scoreHi":167.14,"reasoningOffset":0.85,"rankEligible":false},{"stateId":"qwen3-8-max::effort:high","configSlug":"qwen3-8-max-high-claude-code","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max (high)","modelName":"Qwen3.8-Max","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":2,"score":160.63,"scoreLo":156.69,"scoreHi":165.01,"reasoningOffset":0.5,"rankEligible":false},{"stateId":"qwen3-8-max::effort:low","configSlug":"qwen3-8-max-low-context-arena-mrcr-v2","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max (low)","modelName":"Qwen3.8-Max","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":158.64,"scoreLo":153.93,"scoreHi":164.02,"reasoningOffset":-1.49,"rankEligible":false},{"stateId":"qwen3-8-max::effort:medium","configSlug":"qwen3-8-max-medium-context-arena-mrcr-v2","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max (medium)","modelName":"Qwen3.8-Max","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":159.99,"scoreLo":155.74,"scoreHi":164.83,"reasoningOffset":-0.14,"rankEligible":false},{"stateId":"qwen3-8-max::disabled","configSlug":"qwen3-8-max-no-thinking","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max (disabled)","modelName":"Qwen3.8-Max","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":1,"score":155.37,"scoreLo":148.56,"scoreHi":162.53,"reasoningOffset":-4.75,"rankEligible":false},{"stateId":"qwen3-8-max::enabled","configSlug":"qwen3-8-max-thinking","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max (thinking)","modelName":"Qwen3.8-Max","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":16,"score":160.9,"scoreLo":157.34,"scoreHi":165.08,"reasoningOffset":0.78,"rankEligible":true,"rankMean":39.2617,"rankLo":25.0,"rankHi":53.0},{"stateId":"qwen3-8-max::effort:xhigh","configSlug":"qwen3-8-max-xhigh-context-arena-mrcr-v2","modelId":"qwen3-8-max","modelSlug":"qwen3-8-max","name":"Qwen3.8-Max (xhigh)","modelName":"Qwen3.8-Max","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":2,"score":160.52,"scoreLo":156.74,"scoreHi":165.14,"reasoningOffset":0.39,"rankEligible":false},{"stateId":"qwen3-8-omni-flash::default","configSlug":"qwen3-8-omni-flash","modelId":"qwen3-8-omni-flash","modelSlug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","modelName":"Qwen3.8-Omni-Flash","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":157.96,"scoreLo":154.29,"scoreHi":162.24,"reasoningOffset":0.0,"rankEligible":true,"rankMean":67.5886,"rankLo":47.0,"rankHi":80.0},{"stateId":"qwen3-8b::default","configSlug":"qwen3-8b","modelId":"qwen3-8b","modelSlug":"qwen3-8b","name":"Qwen3 8B","modelName":"Qwen3 8B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":127.97,"scoreLo":125.23,"scoreHi":130.49,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-8b::effort:4000","configSlug":"qwen3-8b-4000","modelId":"qwen3-8b","modelSlug":"qwen3-8b","name":"Qwen3 8B (4000)","modelName":"Qwen3 8B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"4000","reasoningCondition":"effort:4000","isDefault":false,"n":1,"score":126.16,"scoreLo":120.41,"scoreHi":131.03,"reasoningOffset":-1.8,"rankEligible":false},{"stateId":"qwen3-8b-base::default","configSlug":"qwen3-8b-base","modelId":"qwen3-8b-base","modelSlug":"qwen3-8b-base","name":"Qwen3 8B Base","modelName":"Qwen3 8B Base","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":117.33,"scoreLo":112.95,"scoreHi":121.2,"reasoningOffset":0.0,"rankEligible":true,"rankMean":534.9361,"rankLo":500.0,"rankHi":569.0},{"stateId":"qwen3-8b::disabled","configSlug":"qwen3-8b-no-thinking","modelId":"qwen3-8b","modelSlug":"qwen3-8b","name":"Qwen3 8B (disabled)","modelName":"Qwen3 8B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":29,"score":124.32,"scoreLo":121.56,"scoreHi":126.87,"reasoningOffset":-3.65,"rankEligible":true},{"stateId":"qwen3-8b::enabled","configSlug":"qwen3-8b-thinking","modelId":"qwen3-8b","modelSlug":"qwen3-8b","name":"Qwen3 8B (thinking)","modelName":"Qwen3 8B","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":35,"score":129.96,"scoreLo":127.75,"scoreHi":132.02,"reasoningOffset":1.99,"rankEligible":true,"rankMean":375.6375,"rankLo":354.0,"rankHi":397.0},{"stateId":"qwen3-coder-30b-a3b-instruct::default","configSlug":"qwen3-coder-30b-a3b-instruct","modelId":"qwen3-coder-30b-a3b-instruct","modelSlug":"qwen3-coder-30b-a3b-instruct","name":"Qwen3 Coder 30B A3B Instruct","modelName":"Qwen3 Coder 30B A3B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":133.82,"scoreLo":130.58,"scoreHi":136.99,"reasoningOffset":0.0,"rankEligible":true,"rankMean":324.2075,"rankLo":285.0,"rankHi":363.0},{"stateId":"qwen3-coder-480b-a35b-instruct::default","configSlug":"qwen3-coder-480b-a35b-instruct","modelId":"qwen3-coder-480b-a35b-instruct","modelSlug":"qwen3-coder-480b-a35b-instruct","name":"Qwen3-Coder-480B-A35B-Instruct","modelName":"Qwen3-Coder-480B-A35B-Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":33,"score":141.18,"scoreLo":138.91,"scoreHi":143.53,"reasoningOffset":0.0,"rankEligible":true,"rankMean":225.1411,"rankLo":206.0,"rankHi":249.0},{"stateId":"qwen3-coder-next::default","configSlug":"qwen3-coder-next","modelId":"qwen3-coder-next","modelSlug":"qwen3-coder-next","name":"Qwen3-Coder-Next","modelName":"Qwen3-Coder-Next","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":32,"score":140.9,"scoreLo":138.77,"scoreHi":143.19,"reasoningOffset":0.0,"rankEligible":true,"rankMean":228.3503,"rankLo":210.0,"rankHi":251.0},{"stateId":"qwen3-max-2026-01-23::effort:high","configSlug":"qwen3-high","modelId":"qwen3-max-2026-01-23","modelSlug":"qwen3-max-2026-01-23","name":"Qwen3 Max Thinking (2026-01-23) (high)","modelName":"Qwen3 Max Thinking (2026-01-23)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":146.21,"scoreLo":142.8,"scoreHi":149.79,"reasoningOffset":0.87,"rankEligible":false},{"stateId":"qwen3-max-2025-09-23::default","configSlug":"qwen3-max-2025-09-23","modelId":"qwen3-max-2025-09-23","modelSlug":"qwen3-max-2025-09-23","name":"Qwen3-Max (2025-09-23)","modelName":"Qwen3-Max (2025-09-23)","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":21,"score":143.64,"scoreLo":141.06,"scoreHi":146.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":200.2242,"rankLo":178.0,"rankHi":220.0},{"stateId":"qwen3-max-2026-01-23::effort:minimal","configSlug":"qwen3-max-2026-01-23-minimal","modelId":"qwen3-max-2026-01-23","modelSlug":"qwen3-max-2026-01-23","name":"Qwen3 Max Thinking (2026-01-23) (minimal)","modelName":"Qwen3 Max Thinking (2026-01-23)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":141.42,"scoreLo":136.69,"scoreHi":146.54,"reasoningOffset":-3.92,"rankEligible":false},{"stateId":"qwen3-max-2026-01-23::effort:xhigh","configSlug":"qwen3-max-2026-01-23-xhigh","modelId":"qwen3-max-2026-01-23","modelSlug":"qwen3-max-2026-01-23","name":"Qwen3 Max Thinking (2026-01-23) (xhigh)","modelName":"Qwen3 Max Thinking (2026-01-23)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":145.68,"scoreLo":142.6,"scoreHi":149.21,"reasoningOffset":0.34,"rankEligible":false},{"stateId":"qwen3-max-preview::default","configSlug":"qwen3-max-preview","modelId":"qwen3-max-preview","modelSlug":"qwen3-max-preview","name":"Qwen3 Max Preview","modelName":"Qwen3 Max Preview","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":142.04,"scoreLo":139.05,"scoreHi":145.27,"reasoningOffset":0.0,"rankEligible":true,"rankMean":216.4169,"rankLo":188.0,"rankHi":247.025},{"stateId":"qwen3-max-preview::disabled","configSlug":"qwen3-max-preview-no-thinking","modelId":"qwen3-max-preview","modelSlug":"qwen3-max-preview","name":"Qwen3 Max Preview (disabled)","modelName":"Qwen3 Max Preview","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":7,"score":140.68,"scoreLo":137.5,"scoreHi":143.99,"reasoningOffset":-1.36,"rankEligible":true},{"stateId":"qwen3-max-preview::enabled","configSlug":"qwen3-max-preview-thinking","modelId":"qwen3-max-preview","modelSlug":"qwen3-max-preview","name":"Qwen3 Max Preview (thinking)","modelName":"Qwen3 Max Preview","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":11,"score":141.59,"scoreLo":138.77,"scoreHi":144.53,"reasoningOffset":-0.45,"rankEligible":true},{"stateId":"qwen3-max-2026-01-23::enabled","configSlug":"qwen3-max-thinking-2026-01-23","modelId":"qwen3-max-2026-01-23","modelSlug":"qwen3-max-2026-01-23","name":"Qwen3 Max Thinking (2026-01-23) (thinking)","modelName":"Qwen3 Max Thinking (2026-01-23)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":40,"score":147.75,"scoreLo":145.52,"scoreHi":150.33,"reasoningOffset":2.42,"rankEligible":true,"rankMean":156.4492,"rankLo":142.0,"rankHi":170.025},{"stateId":"qwen3-max-2026-01-23::default","configSlug":"qwen3-max-vals-2026-01-23","modelId":"qwen3-max-2026-01-23","modelSlug":"qwen3-max-2026-01-23","name":"Qwen3 Max Thinking (2026-01-23)","modelName":"Qwen3 Max Thinking (2026-01-23)","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":145.34,"scoreLo":142.8,"scoreHi":147.96,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"qwen3-next-80b-a3b-instruct::default","configSlug":"qwen3-next-80b-a3b-instruct","modelId":"qwen3-next-80b-a3b-instruct","modelSlug":"qwen3-next-80b-a3b-instruct","name":"Qwen3 Next 80B A3B Instruct","modelName":"Qwen3 Next 80B A3B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":34,"score":138.7,"scoreLo":136.54,"scoreHi":140.83,"reasoningOffset":0.0,"rankEligible":true,"rankMean":257.6225,"rankLo":234.0,"rankHi":282.0},{"stateId":"qwen3-next-80b-a3b-thinking::default","configSlug":"qwen3-next-80b-a3b-thinking","modelId":"qwen3-next-80b-a3b-thinking","modelSlug":"qwen3-next-80b-a3b-thinking","name":"Qwen3-Next-80B-A3B-Thinking","modelName":"Qwen3-Next-80B-A3B-Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":57,"score":139.21,"scoreLo":137.32,"scoreHi":141.2,"reasoningOffset":2.4,"rankEligible":true,"rankMean":250.5864,"rankLo":231.0,"rankHi":270.0},{"stateId":"qwen3-omni-30b-a3b-instruct::default","configSlug":"qwen3-omni-30b-a3b-instruct","modelId":"qwen3-omni-30b-a3b-instruct","modelSlug":"qwen3-omni-30b-a3b-instruct","name":"Qwen3 Omni 30B A3B Instruct","modelName":"Qwen3 Omni 30B A3B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":131.88,"scoreLo":129.16,"scoreHi":134.48,"reasoningOffset":0.0,"rankEligible":true,"rankMean":349.9928,"rankLo":320.975,"rankHi":380.025},{"stateId":"qwen3-omni-30b-a3b-thinking::default","configSlug":"qwen3-omni-30b-a3b-thinking","modelId":"qwen3-omni-30b-a3b-thinking","modelSlug":"qwen3-omni-30b-a3b-thinking","name":"Qwen3 Omni 30B A3B Thinking","modelName":"Qwen3 Omni 30B A3B Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":30,"score":134.67,"scoreLo":132.18,"scoreHi":137.17,"reasoningOffset":2.4,"rankEligible":true,"rankMean":312.7017,"rankLo":284.0,"rankHi":342.0},{"stateId":"qwen3-vl-235b-a22b-instruct::default","configSlug":"qwen3-vl-235b-a22b","modelId":"qwen3-vl-235b-a22b-instruct","modelSlug":"qwen3-vl-235b-a22b-instruct","name":"Qwen3-VL-235B-A22B-Instruct","modelName":"Qwen3-VL-235B-A22B-Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":62,"score":141.64,"scoreLo":139.63,"scoreHi":143.84,"reasoningOffset":0.0,"rankEligible":true,"rankMean":219.8764,"rankLo":205.0,"rankHi":238.0},{"stateId":"qwen3-vl-235b-a22b-thinking::default","configSlug":"qwen3-vl-235b-a22b-thinking","modelId":"qwen3-vl-235b-a22b-thinking","modelSlug":"qwen3-vl-235b-a22b-thinking","name":"Qwen3 VL 235B A22B Thinking","modelName":"Qwen3 VL 235B A22B Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":64,"score":142.95,"scoreLo":140.76,"scoreHi":145.39,"reasoningOffset":2.4,"rankEligible":true,"rankMean":207.1186,"rankLo":191.0,"rankHi":222.0},{"stateId":"qwen3-vl-2b-instruct::default","configSlug":"qwen3-vl-2b-instruct","modelId":"qwen3-vl-2b-instruct","modelSlug":"qwen3-vl-2b-instruct","name":"Qwen3 VL 2B Instruct","modelName":"Qwen3 VL 2B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":43,"score":113.16,"scoreLo":108.99,"scoreHi":116.75,"reasoningOffset":0.0,"rankEligible":true,"rankMean":584.6678,"rankLo":558.0,"rankHi":611.0},{"stateId":"qwen3-vl-2b-thinking::default","configSlug":"qwen3-vl-2b-thinking","modelId":"qwen3-vl-2b-thinking","modelSlug":"qwen3-vl-2b-thinking","name":"Qwen3 VL 2B Thinking","modelName":"Qwen3 VL 2B Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":43,"score":120.43,"scoreLo":117.11,"scoreHi":123.37,"reasoningOffset":2.4,"rankEligible":true,"rankMean":497.3758,"rankLo":470.0,"rankHi":522.0},{"stateId":"qwen3-vl-30b-a3b-instruct::default","configSlug":"qwen3-vl-30b-a3b-instruct","modelId":"qwen3-vl-30b-a3b-instruct","modelSlug":"qwen3-vl-30b-a3b-instruct","name":"Qwen3 VL 30B A3B Instruct","modelName":"Qwen3 VL 30B A3B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":50,"score":135.1,"scoreLo":133.04,"scoreHi":137.11,"reasoningOffset":0.0,"rankEligible":true,"rankMean":306.6497,"rankLo":285.0,"rankHi":331.0},{"stateId":"qwen3-vl-30b-a3b-thinking::default","configSlug":"qwen3-vl-30b-a3b-thinking","modelId":"qwen3-vl-30b-a3b-thinking","modelSlug":"qwen3-vl-30b-a3b-thinking","name":"Qwen3 VL 30B A3B Thinking","modelName":"Qwen3 VL 30B A3B Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":57,"score":136.99,"scoreLo":134.93,"scoreHi":139.07,"reasoningOffset":2.4,"rankEligible":true,"rankMean":280.9356,"rankLo":258.0,"rankHi":302.0},{"stateId":"qwen3-vl-32b-instruct::default","configSlug":"qwen3-vl-32b-instruct","modelId":"qwen3-vl-32b-instruct","modelSlug":"qwen3-vl-32b-instruct","name":"Qwen3 VL 32B Instruct","modelName":"Qwen3 VL 32B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":50,"score":138.46,"scoreLo":136.49,"scoreHi":140.51,"reasoningOffset":0.0,"rankEligible":true,"rankMean":260.715,"rankLo":239.0,"rankHi":283.025},{"stateId":"qwen3-vl-32b-thinking::default","configSlug":"qwen3-vl-32b-thinking","modelId":"qwen3-vl-32b-thinking","modelSlug":"qwen3-vl-32b-thinking","name":"Qwen3 VL 32B Thinking","modelName":"Qwen3 VL 32B Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":50,"score":140.04,"scoreLo":137.92,"scoreHi":142.25,"reasoningOffset":2.4,"rankEligible":true,"rankMean":239.4122,"rankLo":218.0,"rankHi":262.0},{"stateId":"qwen3-vl-4b-instruct::default","configSlug":"qwen3-vl-4b-instruct","modelId":"qwen3-vl-4b-instruct","modelSlug":"qwen3-vl-4b-instruct","name":"Qwen3 VL 4B Instruct","modelName":"Qwen3 VL 4B Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":48,"score":127.24,"scoreLo":124.64,"scoreHi":129.57,"reasoningOffset":0.0,"rankEligible":true,"rankMean":409.0178,"rankLo":389.0,"rankHi":430.0},{"stateId":"qwen3-vl-4b-thinking::default","configSlug":"qwen3-vl-4b-thinking","modelId":"qwen3-vl-4b-thinking","modelSlug":"qwen3-vl-4b-thinking","name":"Qwen3 VL 4B Thinking","modelName":"Qwen3 VL 4B Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":47,"score":130.16,"scoreLo":127.74,"scoreHi":132.43,"reasoningOffset":2.4,"rankEligible":true,"rankMean":372.8072,"rankLo":349.0,"rankHi":396.025},{"stateId":"qwen3-vl-8b-instruct::default","configSlug":"qwen3-vl-8b-instruct","modelId":"qwen3-vl-8b-instruct","modelSlug":"qwen3-vl-8b-instruct","name":"Qwen3-VL-8B-Instruct","modelName":"Qwen3-VL-8B-Instruct","organizationSlug":"alibaba","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":52,"score":130.48,"scoreLo":128.12,"scoreHi":132.62,"reasoningOffset":0.0,"rankEligible":true,"rankMean":368.4447,"rankLo":346.0,"rankHi":393.0},{"stateId":"qwen3-vl-8b-thinking::default","configSlug":"qwen3-vl-8b-thinking","modelId":"qwen3-vl-8b-thinking","modelSlug":"qwen3-vl-8b-thinking","name":"Qwen3 VL 8B Thinking","modelName":"Qwen3 VL 8B Thinking","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":49,"score":133.6,"scoreLo":131.35,"scoreHi":135.69,"reasoningOffset":2.4,"rankEligible":true,"rankMean":327.6275,"rankLo":302.0,"rankHi":351.0},{"stateId":"qwen3-vl-plus-2025-09-23::default","configSlug":"qwen3-vl-plus-2025-09-23","modelId":"qwen3-vl-plus-2025-09-23","modelSlug":"qwen3-vl-plus-2025-09-23","name":"Qwen3-VL-Plus-2025-09-23","modelName":"Qwen3-VL-Plus-2025-09-23","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":142.74,"scoreLo":133.96,"scoreHi":151.63,"reasoningOffset":0.0,"rankEligible":true,"rankMean":211.88,"rankLo":115.975,"rankHi":321.025},{"stateId":"qwen3-vl-plus-2025-09-23::enabled","configSlug":"qwen3-vl-plus-2025-09-23-thinking","modelId":"qwen3-vl-plus-2025-09-23","modelSlug":"qwen3-vl-plus-2025-09-23","name":"Qwen3-VL-Plus-2025-09-23 (thinking)","modelName":"Qwen3-VL-Plus-2025-09-23","organizationSlug":"alibaba","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":143.68,"scoreLo":134.55,"scoreHi":152.96,"reasoningOffset":0.95,"rankEligible":false},{"stateId":"qwq-32b::default","configSlug":"qwq-32b","modelId":"qwq-32b","modelSlug":"qwq-32b","name":"QwQ-32B","modelName":"QwQ-32B","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"unknown","reasoningCondition":"default","isDefault":true,"n":31,"score":133.64,"scoreLo":131.23,"scoreHi":136.01,"reasoningOffset":2.4,"rankEligible":true,"rankMean":326.9036,"rankLo":298.0,"rankHi":354.0},{"stateId":"qwq-32b::effort:16k","configSlug":"qwq-32b-16k","modelId":"qwq-32b","modelSlug":"qwq-32b","name":"QwQ-32B (16k)","modelName":"QwQ-32B","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"16k","reasoningCondition":"effort:16k","isDefault":false,"n":3,"score":131.73,"scoreLo":128.76,"scoreHi":134.57,"reasoningOffset":0.5,"rankEligible":false},{"stateId":"qwq-32b-preview::default","configSlug":"qwq-32b-preview","modelId":"qwq-32b-preview","modelSlug":"qwq-32b-preview","name":"QwQ-32B-Preview","modelName":"QwQ-32B-Preview","organizationSlug":"alibaba","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":13,"score":118.98,"scoreLo":113.59,"scoreHi":123.71,"reasoningOffset":2.4,"rankEligible":true,"rankMean":514.6694,"rankLo":460.0,"rankHi":564.0},{"stateId":"recurrentgemma-2b::default","configSlug":"recurrentgemma-2b","modelId":"recurrentgemma-2b","modelSlug":"recurrentgemma-2b","name":"RecurrentGemma 2B","modelName":"RecurrentGemma 2B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":28,"score":71.5,"scoreLo":60.46,"scoreHi":80.34,"reasoningOffset":0.0,"rankEligible":true,"rankMean":857.4539,"rankLo":848.0,"rankHi":864.0},{"stateId":"recurrentgemma-9b::default","configSlug":"recurrentgemma-9b","modelId":"recurrentgemma-9b","modelSlug":"recurrentgemma-9b","name":"RecurrentGemma 9B","modelName":"RecurrentGemma 9B","organizationSlug":"google","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":95.3,"scoreLo":88.0,"scoreHi":101.33,"reasoningOffset":0.0,"rankEligible":true,"rankMean":755.8914,"rankLo":727.0,"rankHi":780.0},{"stateId":"redpajama-incite-7b-base::default","configSlug":"redpajama-incite-7b-base","modelId":"redpajama-incite-7b-base","modelSlug":"redpajama-incite-7b-base","name":"RedPajama INCITE 7B Base","modelName":"RedPajama INCITE 7B Base","organizationSlug":"together-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":69.73,"scoreLo":57.6,"scoreHi":80.0,"reasoningOffset":0.0,"rankEligible":true,"rankMean":860.0675,"rankLo":847.0,"rankHi":869.0},{"stateId":"reka-flash-20240226::default","configSlug":"reka-flash-20240226","modelId":"reka-flash-20240226","modelSlug":"reka-flash-20240226","name":"Reka Flash (2024-02-26)","modelName":"Reka Flash (2024-02-26)","organizationSlug":"reka-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":105.18,"scoreLo":98.47,"scoreHi":110.86,"reasoningOffset":0.0,"rankEligible":true,"rankMean":674.1897,"rankLo":630.0,"rankHi":714.0},{"stateId":"reka-flash-3::default","configSlug":"reka-flash-3","modelId":"reka-flash-3","modelSlug":"reka-flash-3","name":"Reka Flash 3","modelName":"Reka Flash 3","organizationSlug":"reka-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":125.78,"scoreLo":120.22,"scoreHi":131.0,"reasoningOffset":2.4,"rankEligible":true,"rankMean":426.8514,"rankLo":363.0,"rankHi":494.025},{"stateId":"ring-1t::default","configSlug":"ring-1t","modelId":"ring-1t","modelSlug":"ring-1t","name":"Ring-1T","modelName":"Ring-1T","organizationSlug":"inclusion-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":136.69,"scoreLo":132.54,"scoreHi":140.66,"reasoningOffset":2.4,"rankEligible":true,"rankMean":285.0467,"rankLo":234.0,"rankHi":339.0},{"stateId":"ring-2-6-1t::default","configSlug":"ring-2-6-1t","modelId":"ring-2-6-1t","modelSlug":"ring-2-6-1t","name":"Ring 2.6 1T","modelName":"Ring 2.6 1T","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":146.87,"scoreLo":144.1,"scoreHi":149.85,"reasoningOffset":0.0,"rankEligible":true,"rankMean":166.0244,"rankLo":143.0,"rankHi":188.0},{"stateId":"ring-2-6-1t::effort:high","configSlug":"ring-2-6-1t-high-context-arena-mrcr-v2","modelId":"ring-2-6-1t","modelSlug":"ring-2-6-1t","name":"Ring 2.6 1T (high)","modelName":"Ring 2.6 1T","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":147.61,"scoreLo":143.89,"scoreHi":151.64,"reasoningOffset":0.74,"rankEligible":false},{"stateId":"ring-2-6-1t::effort:xhigh","configSlug":"ring-2-6-1t-xhigh-context-arena-mrcr-v2","modelId":"ring-2-6-1t","modelSlug":"ring-2-6-1t","name":"Ring 2.6 1T (xhigh)","modelName":"Ring 2.6 1T","organizationSlug":"inclusion-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"xhigh","reasoningCondition":"effort:xhigh","isDefault":false,"n":1,"score":147.85,"scoreLo":144.24,"scoreHi":151.74,"reasoningOffset":0.98,"rankEligible":false},{"stateId":"ring-flash-2-0::default","configSlug":"ring-flash-2-0","modelId":"ring-flash-2-0","modelSlug":"ring-flash-2-0","name":"Ring-flash-2.0","modelName":"Ring-flash-2.0","organizationSlug":"inclusion-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":132.75,"scoreLo":128.25,"scoreHi":137.01,"reasoningOffset":2.4,"rankEligible":true,"rankMean":338.3522,"rankLo":284.975,"rankHi":394.0},{"stateId":"sarvam-105b::effort:high","configSlug":"sarvam-105b-high","modelId":"sarvam-105b","modelSlug":"sarvam-105b","name":"Sarvam 105B (high)","modelName":"Sarvam 105B","organizationSlug":"sarvam-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":5,"score":120.23,"scoreLo":108.05,"scoreHi":129.82,"reasoningOffset":0.51,"rankEligible":true,"rankMean":497.1597,"rankLo":381.975,"rankHi":636.025},{"stateId":"sarvam-30b::effort:high","configSlug":"sarvam-30b-high","modelId":"sarvam-30b","modelSlug":"sarvam-30b","name":"Sarvam 30B (high)","modelName":"Sarvam 30B","organizationSlug":"sarvam-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":5,"score":125.73,"scoreLo":116.13,"scoreHi":133.37,"reasoningOffset":0.51,"rankEligible":true,"rankMean":427.9997,"rankLo":332.0,"rankHi":548.025},{"stateId":"sarvam-m::enabled","configSlug":"sarvam-m-thinking","modelId":"sarvam-m","modelSlug":"sarvam-m","name":"Sarvam M (thinking)","modelName":"Sarvam M","organizationSlug":"sarvam-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":123.57,"scoreLo":117.46,"scoreHi":129.08,"reasoningOffset":1.07,"rankEligible":true,"rankMean":455.0128,"rankLo":393.0,"rankHi":525.025},{"stateId":"seed-oss-36b-instruct::default","configSlug":"seed-oss-36b-instruct","modelId":"seed-oss-36b-instruct","modelSlug":"seed-oss-36b-instruct","name":"Seed OSS 36B Instruct","modelName":"Seed OSS 36B Instruct","organizationSlug":"bytedance","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":138.95,"scoreLo":135.69,"scoreHi":142.16,"reasoningOffset":0.0,"rankEligible":true,"rankMean":254.3169,"rankLo":216.0,"rankHi":295.0},{"stateId":"smaug-72b-v0-1::default","configSlug":"smaug-72b-v0-1","modelId":"smaug-72b-v0-1","modelSlug":"smaug-72b-v0-1","name":"Smaug-72B-v0.1","modelName":"Smaug-72B-v0.1","organizationSlug":"abacus-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":114.98,"scoreLo":110.26,"scoreHi":119.14,"reasoningOffset":0.0,"rankEligible":true,"rankMean":562.3056,"rankLo":528.0,"rankHi":601.0},{"stateId":"lfm2-2-6b::enabled","configSlug":"snakebench-liquidai-lfm2-2-6b","modelId":"lfm2-2-6b","modelSlug":"lfm2-2-6b","name":"LFM2-2.6B (enabled)","modelName":"LFM2-2.6B","organizationSlug":"liquid-ai","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":1,"score":102.61,"scoreLo":93.18,"scoreHi":110.96,"reasoningOffset":1.06,"rankEligible":false},{"stateId":"step-3-7-flash::enabled","configSlug":"snakebench-stepfun-step-3-7-flash","modelId":"step-3-7-flash","modelSlug":"step-3-7-flash","name":"Step 3.7 Flash (enabled)","modelName":"Step 3.7 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":3,"score":149.28,"scoreLo":145.41,"scoreHi":153.55,"reasoningOffset":1.37,"rankEligible":false},{"stateId":"snorkel-mistral-pairrm-dpo::default","configSlug":"snorkel-mistral-pairrm-dpo","modelId":"snorkel-mistral-pairrm-dpo","modelSlug":"snorkel-mistral-pairrm-dpo","name":"Snorkel-Mistral-PairRM-DPO","modelName":"Snorkel-Mistral-PairRM-DPO","organizationSlug":"snorkel-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":99.92,"scoreLo":92.57,"scoreHi":106.13,"reasoningOffset":0.0,"rankEligible":true,"rankMean":719.0761,"rankLo":682.0,"rankHi":756.0},{"stateId":"solar-10-7b-instruct-v1-0::default","configSlug":"solar-10-7b-instruct-v1-0","modelId":"solar-10-7b-instruct-v1-0","modelSlug":"solar-10-7b-instruct-v1-0","name":"Solar 10.7B Instruct V1.0","modelName":"Solar 10.7B Instruct V1.0","organizationSlug":"upstage","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":100.88,"scoreLo":95.05,"scoreHi":105.74,"reasoningOffset":0.0,"rankEligible":true,"rankMean":711.7594,"rankLo":697.0,"rankHi":725.0},{"stateId":"solar-mini4-260922::enabled","configSlug":"solar-mini4-260922-thinking","modelId":"solar-mini4-260922","modelSlug":"solar-mini4-260922","name":"Solar Mini 4 (thinking)","modelName":"Solar Mini 4","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":8,"score":150.43,"scoreLo":146.96,"scoreHi":154.32,"reasoningOffset":1.07,"rankEligible":true,"rankMean":127.2647,"rankLo":102.0,"rankHi":158.0},{"stateId":"solar-open-100b::default","configSlug":"solar-open-100b-thinking","modelId":"solar-open-100b","modelSlug":"solar-open-100b","name":"Solar Open 100B","modelName":"Solar Open 100B","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":6,"score":132.69,"scoreLo":127.72,"scoreHi":137.4,"reasoningOffset":0.0,"rankEligible":true,"rankMean":339.1322,"rankLo":279.0,"rankHi":402.0},{"stateId":"solar-open2-250b::enabled","configSlug":"solar-open2-250b-thinking","modelId":"solar-open2-250b","modelSlug":"solar-open2-250b","name":"Solar Open2 250B (thinking)","modelName":"Solar Open2 250B","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":5,"score":150.36,"scoreLo":146.19,"scoreHi":154.91,"reasoningOffset":1.07,"rankEligible":true,"rankMean":128.3114,"rankLo":95.0,"rankHi":168.0},{"stateId":"solar-pro-preview-instruct::default","configSlug":"solar-pro-preview-instruct","modelId":"solar-pro-preview-instruct","modelSlug":"solar-pro-preview-instruct","name":"Solar Pro Instruct Preview","modelName":"Solar Pro Instruct Preview","organizationSlug":"upstage","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":8,"score":121.08,"scoreLo":116.61,"scoreHi":125.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":487.885,"rankLo":440.0,"rankHi":533.025},{"stateId":"solar-pro2-250710::disabled","configSlug":"solar-pro2-250710-no-thinking","modelId":"solar-pro2-250710","modelSlug":"solar-pro2-250710","name":"Solar Pro 2 (disabled)","modelName":"Solar Pro 2","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":8,"score":129.53,"scoreLo":125.78,"scoreHi":133.09,"reasoningOffset":-3.99,"rankEligible":true},{"stateId":"solar-pro2-250710::enabled","configSlug":"solar-pro2-250710-thinking","modelId":"solar-pro2-250710","modelSlug":"solar-pro2-250710","name":"Solar Pro 2 (thinking)","modelName":"Solar Pro 2","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":8,"score":134.49,"scoreLo":131.2,"scoreHi":137.8,"reasoningOffset":0.97,"rankEligible":true,"rankMean":315.1617,"rankLo":273.0,"rankHi":355.0},{"stateId":"solar-pro3-260126::default","configSlug":"solar-pro3-260126","modelId":"solar-pro3-260126","modelSlug":"solar-pro3-260126","name":"Solar Pro 3","modelName":"Solar Pro 3","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":18,"score":131.29,"scoreLo":127.42,"scoreHi":134.93,"reasoningOffset":0.0,"rankEligible":true,"rankMean":357.6114,"rankLo":314.0,"rankHi":403.0},{"stateId":"solar-pro4::default","configSlug":"solar-pro4","modelId":"solar-pro4","modelSlug":"solar-pro4","name":"Solar Pro 4","modelName":"Solar Pro 4","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"unknown","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":143.14,"scoreLo":138.87,"scoreHi":147.53,"reasoningOffset":0.0,"rankEligible":true,"rankMean":205.3372,"rankLo":165.0,"rankHi":249.025},{"stateId":"solar-pro4::enabled","configSlug":"solar-pro4-thinking-context-arena-mrcr-v2","modelId":"solar-pro4","modelSlug":"solar-pro4","name":"Solar Pro 4 (thinking)","modelName":"Solar Pro 4","organizationSlug":"upstage","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":1,"score":144.45,"scoreLo":139.38,"scoreHi":149.78,"reasoningOffset":1.3,"rankEligible":false},{"stateId":"sonar::default","configSlug":"sonar","modelId":"sonar","modelSlug":"sonar","name":"Sonar","modelName":"Sonar","organizationSlug":"perplexity","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":125.92,"scoreLo":119.55,"scoreHi":131.9,"reasoningOffset":0.0,"rankEligible":true,"rankMean":425.3756,"rankLo":353.0,"rankHi":504.0},{"stateId":"sonar-pro::default","configSlug":"sonar-pro","modelId":"sonar-pro","modelSlug":"sonar-pro","name":"Sonar Pro","modelName":"Sonar Pro","organizationSlug":"perplexity","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":130.99,"scoreLo":124.87,"scoreHi":137.03,"reasoningOffset":0.0,"rankEligible":true,"rankMean":361.2425,"rankLo":281.975,"rankHi":433.0},{"stateId":"claude-3-7-sonnet-20250219::disabled","configSlug":"sonnet-3-7-no-thinking","modelId":"claude-3-7-sonnet-20250219","modelSlug":"claude-3-7-sonnet-20250219","name":"Claude 3.7 Sonnet (disabled)","modelName":"Claude 3.7 Sonnet","organizationSlug":"anthropic","reasoningSupport":"optional","reasoningMode":"disabled","reasoningEffort":"none","reasoningCondition":"disabled","isDefault":false,"n":37,"score":137.69,"scoreLo":135.75,"scoreHi":139.61,"reasoningOffset":-2.68,"rankEligible":true},{"stateId":"stablebeluga2::default","configSlug":"stablebeluga2","modelId":"stablebeluga2","modelSlug":"stablebeluga2","name":"Stable Beluga 2","modelName":"Stable Beluga 2","organizationSlug":"stability-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":104.46,"scoreLo":98.76,"scoreHi":109.33,"reasoningOffset":0.0,"rankEligible":true,"rankMean":681.35,"rankLo":654.0,"rankHi":707.0},{"stateId":"stablelm-2-zephyr-1-6b::default","configSlug":"stablelm-2-zephyr-1-6b","modelId":"stablelm-2-zephyr-1-6b","modelSlug":"stablelm-2-zephyr-1-6b","name":"StableLM 2 Zephyr 1.6B","modelName":"StableLM 2 Zephyr 1.6B","organizationSlug":"stability-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":75.65,"scoreLo":64.49,"scoreHi":84.95,"reasoningOffset":0.0,"rankEligible":true,"rankMean":846.7139,"rankLo":829.0,"rankHi":861.0},{"stateId":"stablelm-tuned-alpha-7b::default","configSlug":"stablelm-tuned-alpha-7b","modelId":"stablelm-tuned-alpha-7b","modelSlug":"stablelm-tuned-alpha-7b","name":"StableLM Tuned Alpha 7B","modelName":"StableLM Tuned Alpha 7B","organizationSlug":"stability-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":54.29,"scoreLo":39.02,"scoreHi":67.51,"reasoningOffset":0.0,"rankEligible":true,"rankMean":877.07,"rankLo":870.0,"rankHi":883.0},{"stateId":"starcoder2-15b::default","configSlug":"starcoder2-15b","modelId":"starcoder2-15b","modelSlug":"starcoder2-15b","name":"StarCoder2-15B","modelName":"StarCoder2-15B","organizationSlug":"hugging-face","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":90.69,"scoreLo":82.92,"scoreHi":97.28,"reasoningOffset":0.0,"rankEligible":true,"rankMean":786.1311,"rankLo":767.0,"rankHi":804.0},{"stateId":"starcoder2-3b::default","configSlug":"starcoder2-3b","modelId":"starcoder2-3b","modelSlug":"starcoder2-3b","name":"StarCoder2-3B","modelName":"StarCoder2-3B","organizationSlug":"hugging-face","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":71.84,"scoreLo":60.46,"scoreHi":80.93,"reasoningOffset":0.0,"rankEligible":true,"rankMean":856.5022,"rankLo":845.0,"rankHi":865.0},{"stateId":"starcoder2-7b::default","configSlug":"starcoder2-7b","modelId":"starcoder2-7b","modelSlug":"starcoder2-7b","name":"StarCoder2-7B","modelName":"StarCoder2-7B","organizationSlug":"hugging-face","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":26,"score":77.34,"scoreLo":67.12,"scoreHi":85.59,"reasoningOffset":0.0,"rankEligible":true,"rankMean":842.0403,"rankLo":829.0,"rankHi":855.0},{"stateId":"step-3-5-flash::default","configSlug":"step-3-5-flash","modelId":"step-3-5-flash","modelSlug":"step-3-5-flash","name":"Step 3.5 Flash","modelName":"Step 3.5 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":146.17,"scoreLo":142.72,"scoreHi":149.75,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"step-3-5-flash-2603::default","configSlug":"step-3-5-flash-2603","modelId":"step-3-5-flash-2603","modelSlug":"step-3-5-flash-2603","name":"Step 3.5 Flash 2603","modelName":"Step 3.5 Flash 2603","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":5,"score":148.23,"scoreLo":143.51,"scoreHi":153.02,"reasoningOffset":0.0,"rankEligible":true,"rankMean":151.1303,"rankLo":109.0,"rankHi":197.0},{"stateId":"step-3-5-flash::effort:minimal","configSlug":"step-3-5-flash-minimal","modelId":"step-3-5-flash","modelSlug":"step-3-5-flash","name":"Step 3.5 Flash (minimal)","modelName":"Step 3.5 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":142.03,"scoreLo":136.65,"scoreHi":147.69,"reasoningOffset":-4.14,"rankEligible":false},{"stateId":"step-3-5-flash::enabled","configSlug":"step-3-5-flash-thinking","modelId":"step-3-5-flash","modelSlug":"step-3-5-flash","name":"Step 3.5 Flash (thinking)","modelName":"Step 3.5 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":16,"score":149.02,"scoreLo":146.08,"scoreHi":152.2,"reasoningOffset":2.85,"rankEligible":true,"rankMean":142.3981,"rankLo":119.0,"rankHi":167.0},{"stateId":"step-3-7-flash::default","configSlug":"step-3-7-flash","modelId":"step-3-7-flash","modelSlug":"step-3-7-flash","name":"Step 3.7 Flash","modelName":"Step 3.7 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":17,"score":147.9,"scoreLo":145.26,"scoreHi":150.89,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"step-3-7-flash::effort:high","configSlug":"step-3-7-flash-high","modelId":"step-3-7-flash","modelSlug":"step-3-7-flash","name":"Step 3.7 Flash (high)","modelName":"Step 3.7 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":12,"score":149.11,"scoreLo":146.56,"scoreHi":151.94,"reasoningOffset":1.21,"rankEligible":true,"rankMean":141.3289,"rankLo":122.0,"rankHi":160.0},{"stateId":"step-3-7-flash::effort:low","configSlug":"step-3-7-flash-low-context-arena-mrcr-v2","modelId":"step-3-7-flash","modelSlug":"step-3-7-flash","name":"Step 3.7 Flash (low)","modelName":"Step 3.7 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"low","reasoningCondition":"effort:low","isDefault":false,"n":1,"score":145.59,"scoreLo":141.11,"scoreHi":150.32,"reasoningOffset":-2.32,"rankEligible":false},{"stateId":"step-3-7-flash::effort:medium","configSlug":"step-3-7-flash-medium-ale-bench-self-refine-x1","modelId":"step-3-7-flash","modelSlug":"step-3-7-flash","name":"Step 3.7 Flash (medium)","modelName":"Step 3.7 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"medium","reasoningCondition":"effort:medium","isDefault":false,"n":2,"score":147.18,"scoreLo":143.19,"scoreHi":151.35,"reasoningOffset":-0.72,"rankEligible":false},{"stateId":"step-3-7-flash::effort:minimal","configSlug":"step-3-7-flash-minimal","modelId":"step-3-7-flash","modelSlug":"step-3-7-flash","name":"Step 3.7 Flash (minimal)","modelName":"Step 3.7 Flash","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"minimal","reasoningCondition":"effort:minimal","isDefault":false,"n":1,"score":143.84,"scoreLo":139.0,"scoreHi":149.19,"reasoningOffset":-4.07,"rankEligible":false},{"stateId":"step-5-preview::default","configSlug":"step-5-preview","modelId":"step-5-preview","modelSlug":"step-5-preview","name":"Step 5 Preview","modelName":"Step 5 Preview","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":40,"score":159.44,"scoreLo":156.25,"scoreHi":163.22,"reasoningOffset":0.0,"rankEligible":true},{"stateId":"step-5-preview::effort:high","configSlug":"step-5-preview-high","modelId":"step-5-preview","modelSlug":"step-5-preview","name":"Step 5 Preview (high)","modelName":"Step 5 Preview","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":14,"score":159.93,"scoreLo":156.47,"scoreHi":164.0,"reasoningOffset":0.49,"rankEligible":true,"rankMean":49.6678,"rankLo":33.0,"rankHi":65.0},{"stateId":"step3-vl-10b::default","configSlug":"step3-vl-10b","modelId":"step3-vl-10b","modelSlug":"step3-vl-10b","name":"Step3-VL-10B","modelName":"Step3-VL-10B","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":136.01,"scoreLo":131.16,"scoreHi":140.49,"reasoningOffset":0.0,"rankEligible":true,"rankMean":294.32,"rankLo":236.975,"rankHi":355.0},{"stateId":"step3-vl-10b::enabled","configSlug":"step3-vl-10b-aa-tau2-bench-telecom","modelId":"step3-vl-10b","modelSlug":"step3-vl-10b","name":"Step3-VL-10B (enabled)","modelName":"Step3-VL-10B","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":null,"reasoningCondition":"enabled","isDefault":false,"n":2,"score":136.35,"scoreLo":131.0,"scoreHi":141.43,"reasoningOffset":0.34,"rankEligible":false},{"stateId":"step3-vl-10b::effort:pacore","configSlug":"step3-vl-10b-pacore","modelId":"step3-vl-10b","modelSlug":"step3-vl-10b","name":"Step3-VL-10B (pacore)","modelName":"Step3-VL-10B","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"pacore","reasoningCondition":"effort:pacore","isDefault":false,"n":1,"score":139.27,"scoreLo":134.75,"scoreHi":143.95,"reasoningOffset":3.26,"rankEligible":false},{"stateId":"step3-vl-10b::effort:sere","configSlug":"step3-vl-10b-sere","modelId":"step3-vl-10b","modelSlug":"step3-vl-10b","name":"Step3-VL-10B (sere)","modelName":"Step3-VL-10B","organizationSlug":"stepfun","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"sere","reasoningCondition":"effort:sere","isDefault":false,"n":1,"score":138.27,"scoreLo":133.8,"scoreHi":142.85,"reasoningOffset":2.26,"rankEligible":false},{"stateId":"ternary-bonsai-2-27b-gguf::enabled","configSlug":"ternary-bonsai-2-27b-gguf-thinking-evalscope-vllm","modelId":"ternary-bonsai-2-27b-gguf","modelSlug":"ternary-bonsai-2-27b-gguf","name":"Ternary Bonsai 2 27B (GGUF) (thinking)","modelName":"Ternary Bonsai 2 27B (GGUF)","organizationSlug":"prism-ml","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":4,"score":151.06,"scoreLo":144.39,"scoreHi":158.55,"reasoningOffset":1.07,"rankEligible":true,"rankMean":123.1408,"rankLo":68.0,"rankHi":189.0},{"stateId":"tiny-aya-global::default","configSlug":"tiny-aya-global","modelId":"tiny-aya-global","modelSlug":"tiny-aya-global","name":"Tiny Aya Global","modelName":"Tiny Aya Global","organizationSlug":"cohere","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":105.97,"scoreLo":95.11,"scoreHi":114.72,"reasoningOffset":0.0,"rankEligible":true,"rankMean":663.495,"rankLo":574.0,"rankHi":749.0},{"stateId":"tri-21b-think::default","configSlug":"tri-21b-think","modelId":"tri-21b-think","modelSlug":"tri-21b-think","name":"Tri-21B-Think","modelName":"Tri-21B-Think","organizationSlug":"trillion-labs","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":5,"score":126.42,"scoreLo":119.9,"scoreHi":132.0,"reasoningOffset":2.4,"rankEligible":true,"rankMean":419.0467,"rankLo":349.0,"rankHi":500.0},{"stateId":"tri-21b-think-preview::default","configSlug":"tri-21b-think-preview","modelId":"tri-21b-think-preview","modelSlug":"tri-21b-think-preview","name":"Tri-21B-Think Preview","modelName":"Tri-21B-Think Preview","organizationSlug":"trillion-labs","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":5,"score":127.42,"scoreLo":119.82,"scoreHi":133.95,"reasoningOffset":2.4,"rankEligible":true,"rankMean":406.6025,"rankLo":325.975,"rankHi":502.0},{"stateId":"trinity-large-thinking::default","configSlug":"trinity-large-thinking","modelId":"trinity-large-thinking","modelSlug":"trinity-large-thinking","name":"Trinity Large Thinking","modelName":"Trinity Large Thinking","organizationSlug":"arcee-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"thinking","reasoningCondition":"default","isDefault":true,"n":27,"score":140.07,"scoreLo":137.77,"scoreHi":142.44,"reasoningOffset":2.4,"rankEligible":true,"rankMean":238.9444,"rankLo":216.0,"rankHi":264.0},{"stateId":"trinity-large-thinking::effort:high","configSlug":"trinity-large-thinking-high","modelId":"trinity-large-thinking","modelSlug":"trinity-large-thinking","name":"Trinity Large Thinking (high)","modelName":"Trinity Large Thinking","organizationSlug":"arcee-ai","reasoningSupport":"always_on","reasoningMode":"default","reasoningEffort":"high","reasoningCondition":"effort:high","isDefault":false,"n":1,"score":136.97,"scoreLo":133.45,"scoreHi":140.57,"reasoningOffset":-0.71,"rankEligible":false},{"stateId":"tulu-2-dpo-70b::default","configSlug":"tulu-2-dpo-70b","modelId":"tulu-2-dpo-70b","modelSlug":"tulu-2-dpo-70b","name":"T\u00fclu 2 DPO 70B","modelName":"T\u00fclu 2 DPO 70B","organizationSlug":"allen-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":107.69,"scoreLo":101.74,"scoreHi":112.86,"reasoningOffset":0.0,"rankEligible":true,"rankMean":648.9456,"rankLo":606.975,"rankHi":686.0},{"stateId":"vicuna-13b-v1-1::default","configSlug":"vicuna-13b-v1-1","modelId":"vicuna-13b-v1-1","modelSlug":"vicuna-13b-v1-1","name":"Vicuna 13B v1.1","modelName":"Vicuna 13B v1.1","organizationSlug":"lmsys","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":10,"score":88.38,"scoreLo":79.44,"scoreHi":95.84,"reasoningOffset":0.0,"rankEligible":true,"rankMean":798.2781,"rankLo":772.0,"rankHi":821.025},{"stateId":"vicuna-13b-v1-3::default","configSlug":"vicuna-13b-v1-3","modelId":"vicuna-13b-v1-3","modelSlug":"vicuna-13b-v1-3","name":"Vicuna 13B v1.3","modelName":"Vicuna 13B v1.3","organizationSlug":"lmsys","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":89.05,"scoreLo":80.3,"scoreHi":96.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":794.8297,"rankLo":769.0,"rankHi":817.0},{"stateId":"vicuna-13b-v1-5::default","configSlug":"vicuna-13b-v1-5","modelId":"vicuna-13b-v1-5","modelSlug":"vicuna-13b-v1-5","name":"Vicuna 13B v1.5","modelName":"Vicuna 13B v1.5","organizationSlug":"lmsys","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":95.06,"scoreLo":86.92,"scoreHi":101.98,"reasoningOffset":0.0,"rankEligible":true,"rankMean":757.1036,"rankLo":719.975,"rankHi":790.0},{"stateId":"vicuna-33b-v1-3::default","configSlug":"vicuna-33b-v1-3","modelId":"vicuna-33b-v1-3","modelSlug":"vicuna-33b-v1-3","name":"Vicuna 33B v1.3","modelName":"Vicuna 33B v1.3","organizationSlug":"lmsys","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":97.94,"scoreLo":90.11,"scoreHi":104.31,"reasoningOffset":0.0,"rankEligible":true,"rankMean":734.8508,"rankLo":700.0,"rankHi":770.0},{"stateId":"vicuna-7b-v1-1::default","configSlug":"vicuna-7b-v1-1","modelId":"vicuna-7b-v1-1","modelSlug":"vicuna-7b-v1-1","name":"Vicuna 7B v1.1","modelName":"Vicuna 7B v1.1","organizationSlug":"lmsys","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":73.59,"scoreLo":58.28,"scoreHi":86.67,"reasoningOffset":0.0,"rankEligible":true,"rankMean":850.0175,"rankLo":818.0,"rankHi":871.0},{"stateId":"wizardlm-2-7b::default","configSlug":"wizardlm-2-7b","modelId":"wizardlm-2-7b","modelSlug":"wizardlm-2-7b","name":"WizardLM-2 7B","modelName":"WizardLM-2 7B","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":7,"score":99.22,"scoreLo":91.71,"scoreHi":105.78,"reasoningOffset":0.0,"rankEligible":true,"rankMean":724.6336,"rankLo":687.0,"rankHi":762.0},{"stateId":"wizardlm-2-8x22b::default","configSlug":"wizardlm-2-8x22b","modelId":"wizardlm-2-8x22b","modelSlug":"wizardlm-2-8x22b","name":"WizardLM-2 8x22B","modelName":"WizardLM-2 8x22B","organizationSlug":"microsoft","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":6,"score":116.44,"scoreLo":110.41,"scoreHi":121.94,"reasoningOffset":0.0,"rankEligible":true,"rankMean":545.3264,"rankLo":485.0,"rankHi":606.0},{"stateId":"mimo-v2-omni::enabled","configSlug":"xiaomi-mimo-v2-omni-thinking","modelId":"mimo-v2-omni","modelSlug":"mimo-v2-omni","name":"MiMo-V2-Omni (thinking)","modelName":"MiMo-V2-Omni","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":7,"score":147.54,"scoreLo":144.07,"scoreHi":151.3,"reasoningOffset":0.87,"rankEligible":true,"rankMean":158.6078,"rankLo":126.0,"rankHi":190.0},{"stateId":"mimo-v2-pro::enabled","configSlug":"xiaomi-mimo-v2-pro-thinking","modelId":"mimo-v2-pro","modelSlug":"mimo-v2-pro","name":"MiMo-V2-Pro (thinking)","modelName":"MiMo-V2-Pro","organizationSlug":"xiaomi","reasoningSupport":"optional","reasoningMode":"enabled","reasoningEffort":"thinking","reasoningCondition":"enabled","isDefault":false,"n":8,"score":146.68,"scoreLo":143.65,"scoreHi":149.85,"reasoningOffset":0.44,"rankEligible":true,"rankMean":168.0267,"rankLo":142.0,"rankHi":194.0},{"stateId":"yi-1-5-34b-chat::default","configSlug":"yi-1-5-34b-chat","modelId":"yi-1-5-34b-chat","modelSlug":"yi-1-5-34b-chat","name":"Yi 1.5 34B Chat","modelName":"Yi 1.5 34B Chat","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":27,"score":113.04,"scoreLo":108.58,"scoreHi":116.73,"reasoningOffset":0.0,"rankEligible":true,"rankMean":586.09,"rankLo":558.0,"rankHi":614.0},{"stateId":"yi-1-5-6b-chat::default","configSlug":"yi-1-5-6b-chat","modelId":"yi-1-5-6b-chat","modelSlug":"yi-1-5-6b-chat","name":"Yi 1.5 6B Chat","modelName":"Yi 1.5 6B Chat","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":16,"score":101.79,"scoreLo":95.2,"scoreHi":107.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":704.1222,"rankLo":675.0,"rankHi":730.0},{"stateId":"yi-1-5-9b-chat::default","configSlug":"yi-1-5-9b-chat","modelId":"yi-1-5-9b-chat","modelSlug":"yi-1-5-9b-chat","name":"Yi 1.5 9B Chat","modelName":"Yi 1.5 9B Chat","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":22,"score":108.79,"scoreLo":103.76,"scoreHi":112.86,"reasoningOffset":0.0,"rankEligible":true,"rankMean":637.3389,"rankLo":612.0,"rankHi":663.0},{"stateId":"yi-34b::default","configSlug":"yi-34b","modelId":"yi-34b","modelSlug":"yi-34b","name":"Yi-34B","modelName":"Yi-34B","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":15,"score":111.38,"scoreLo":106.36,"scoreHi":115.88,"reasoningOffset":0.0,"rankEligible":true,"rankMean":606.5906,"rankLo":564.0,"rankHi":645.0},{"stateId":"yi-34b-chat::default","configSlug":"yi-34b-chat","modelId":"yi-34b-chat","modelSlug":"yi-34b-chat","name":"Yi 34B Chat","modelName":"Yi 34B Chat","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":23,"score":107.35,"scoreLo":101.96,"scoreHi":111.71,"reasoningOffset":0.0,"rankEligible":true,"rankMean":653.1597,"rankLo":625.0,"rankHi":680.0},{"stateId":"yi-6b::default","configSlug":"yi-6b","modelId":"yi-6b","modelSlug":"yi-6b","name":"Yi-6B","modelName":"Yi-6B","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":97.48,"scoreLo":90.31,"scoreHi":103.41,"reasoningOffset":0.0,"rankEligible":true,"rankMean":738.5531,"rankLo":710.0,"rankHi":768.0},{"stateId":"yi-6b-chat::default","configSlug":"yi-6b-chat","modelId":"yi-6b-chat","modelSlug":"yi-6b-chat","name":"Yi 6B Chat","modelName":"Yi 6B Chat","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":9,"score":95.74,"scoreLo":87.81,"scoreHi":102.58,"reasoningOffset":0.0,"rankEligible":true,"rankMean":752.1881,"rankLo":719.0,"rankHi":784.0},{"stateId":"yi-9b::default","configSlug":"yi-9b","modelId":"yi-9b","modelSlug":"yi-9b","name":"Yi-9B","modelName":"Yi-9B","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":11,"score":103.92,"scoreLo":97.8,"scoreHi":109.32,"reasoningOffset":0.0,"rankEligible":true,"rankMean":685.9678,"rankLo":650.975,"rankHi":718.0},{"stateId":"yi-large::default","configSlug":"yi-large","modelId":"yi-large","modelSlug":"yi-large","name":"Yi-Large","modelName":"Yi-Large","organizationSlug":"01-ai","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":4,"score":118.72,"scoreLo":113.27,"scoreHi":123.96,"reasoningOffset":0.0,"rankEligible":true,"rankMean":517.8942,"rankLo":455.0,"rankHi":571.0},{"stateId":"zephyr-7b-alpha::default","configSlug":"zephyr-7b-alpha","modelId":"zephyr-7b-alpha","modelSlug":"zephyr-7b-alpha","name":"Zephyr 7B Alpha","modelName":"Zephyr 7B Alpha","organizationSlug":"hugging-face","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":13,"score":96.84,"scoreLo":89.81,"scoreHi":102.9,"reasoningOffset":0.0,"rankEligible":true,"rankMean":743.7353,"rankLo":716.0,"rankHi":771.0},{"stateId":"zephyr-7b-beta::default","configSlug":"zephyr-7b-beta","modelId":"zephyr-7b-beta","modelSlug":"zephyr-7b-beta","name":"Zephyr 7B Beta","modelName":"Zephyr 7B Beta","organizationSlug":"hugging-face","reasoningSupport":"none","reasoningMode":"default","reasoningEffort":null,"reasoningCondition":"default","isDefault":true,"n":25,"score":96.82,"scoreLo":90.22,"scoreHi":102.15,"reasoningOffset":0.0,"rankEligible":true,"rankMean":744.1089,"rankLo":729.0,"rankHi":759.0}],"benchmarkAvailability":[{"slug":"graphwalks-bfs-256k","signalToNoise":{"available":true,"value":9.247},"difficulty":{"available":true,"value":144.9}},{"slug":"graphwalks-bfs-1m","signalToNoise":{"available":true,"value":7.189},"difficulty":{"available":true,"value":158.5}},{"slug":"mmlu","signalToNoise":{"available":true,"value":6.338},"difficulty":{"available":true,"value":99.8}},{"slug":"humaneval","signalToNoise":{"available":true,"value":2.283},"difficulty":{"available":true,"value":99.7}},{"slug":"math","signalToNoise":{"available":true,"value":3.07},"difficulty":{"available":true,"value":111.6}},{"slug":"code-migration","signalToNoise":{"available":true,"value":10.328},"difficulty":{"available":true,"value":163.0}},{"slug":"corp_fin_v2","signalToNoise":{"available":true,"value":3.576},"difficulty":{"available":true,"value":130.7}},{"slug":"gpqa-diamond","signalToNoise":{"available":true,"value":9.474},"difficulty":{"available":true,"value":133.1}},{"slug":"gpqa-diamond-average-5-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab","signalToNoise":{"available":true,"value":5.102},"difficulty":{"available":true,"value":188.6}},{"slug":"ioi","signalToNoise":{"available":true,"value":11.801},"difficulty":{"available":true,"value":156.5}},{"slug":"legal_bench","signalToNoise":{"available":true,"value":2.213},"difficulty":{"available":true,"value":85.3}},{"slug":"medcode","signalToNoise":{"available":true,"value":2.818},"difficulty":{"available":true,"value":163.6}},{"slug":"medscribe","signalToNoise":{"available":true,"value":3.647},"difficulty":{"available":true,"value":116.2}},{"slug":"mortgage_tax","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"mortgage_tax-vals-all-1258","signalToNoise":{"available":true,"value":1.717},"difficulty":{"available":true,"value":124.7}},{"slug":"mystery-game-puzzles","signalToNoise":{"available":true,"value":4.351},"difficulty":{"available":true,"value":167.6}},{"slug":"programbench","signalToNoise":{"available":true,"value":33.729},"difficulty":{"available":true,"value":184.8}},{"slug":"proof_bench","signalToNoise":{"available":true,"value":11.52},"difficulty":{"available":true,"value":159.0}},{"slug":"public-benefits-bench","signalToNoise":{"available":true,"value":5.229},"difficulty":{"available":true,"value":143.9}},{"slug":"sage","signalToNoise":{"available":true,"value":2.055},"difficulty":{"available":true,"value":167.1}},{"slug":"tax_eval_v2","signalToNoise":{"available":true,"value":2.936},"difficulty":{"available":true,"value":102.6}},{"slug":"vectara-hallucination-leaderboard","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"vibe-code","signalToNoise":{"available":true,"value":12.785},"difficulty":{"available":true,"value":155.1}},{"slug":"poker_agent","signalToNoise":{"available":true,"value":4.992},"difficulty":{"available":true,"value":145.7}},{"slug":"otis-mock-aime-2024-2025","signalToNoise":{"available":true,"value":5.254},"difficulty":{"available":true,"value":134.5}},{"slug":"chess-puzzles","signalToNoise":{"available":true,"value":5.545},"difficulty":{"available":true,"value":167.3}},{"slug":"chess-puzzles-v2","signalToNoise":{"available":true,"value":4.71},"difficulty":{"available":true,"value":164.8}},{"slug":"simple-qa-verified","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-verified","signalToNoise":{"available":true,"value":5.422},"difficulty":{"available":true,"value":136.9}},{"slug":"math-level-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aider-polyglot","signalToNoise":{"available":true,"value":8.298},"difficulty":{"available":true,"value":138.1}},{"slug":"balrog","signalToNoise":{"available":true,"value":3.899},"difficulty":{"available":true,"value":154.4}},{"slug":"balrog-babyai","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"balrog-crafter","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"balrog-textworld","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"balrog-babaisai","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"balrog-minihack","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"balrog-nethack","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"balrog-vlm","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"balrog-vlm-babyai","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"balrog-vlm-crafter","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"balrog-vlm-babaisai","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"balrog-vlm-minihack","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"balrog-vlm-nethack","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"weirdml","signalToNoise":{"available":true,"value":7.449},"difficulty":{"available":true,"value":151.0}},{"slug":"weirdml-v3","signalToNoise":{"available":true,"value":20.163},"difficulty":{"available":true,"value":173.3}},{"slug":"vpct","signalToNoise":{"available":true,"value":17.785},"difficulty":{"available":true,"value":153.3}},{"slug":"fictionlivebench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"fictionlivebench-0-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-400-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-1k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-2k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-4k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-8k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-16k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-32k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-60k-tokens","signalToNoise":{"available":true,"value":5.194},"difficulty":{"available":true,"value":136.7}},{"slug":"fictionlivebench-120k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"fictionlivebench-192k-tokens","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"geobench","signalToNoise":{"available":true,"value":1.517},"difficulty":{"available":true,"value":107.8}},{"slug":"geobench-acw-country-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"geobench-photos","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"geobench-photos-country-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"simplebench","signalToNoise":{"available":true,"value":6.298},"difficulty":{"available":true,"value":153.4}},{"slug":"gdpval","signalToNoise":{"available":true,"value":10.793},"difficulty":{"available":true,"value":147.9}},{"slug":"metr-time-horizons","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"deepresearchbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"frontierswe","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena","signalToNoise":{"available":true,"value":7.631},"difficulty":{"available":true,"value":153.6}},{"slug":"adversarial-nli","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"arc-ai2","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"bbh","signalToNoise":{"available":true,"value":8.179},"difficulty":{"available":true,"value":105.1}},{"slug":"bool-q","signalToNoise":{"available":true,"value":3.185},"difficulty":{"available":true,"value":86.0}},{"slug":"cad-eval","signalToNoise":{"available":true,"value":5.782},"difficulty":{"available":true,"value":139.4}},{"slug":"common-sense-qa-2","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"cybench","signalToNoise":{"available":true,"value":7.115},"difficulty":{"available":true,"value":145.4}},{"slug":"gsm8k","signalToNoise":{"available":true,"value":4.107},"difficulty":{"available":true,"value":91.6}},{"slug":"lambada","signalToNoise":{"available":true,"value":2.74},"difficulty":{"available":true,"value":32.7}},{"slug":"lech-mazur-writing","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"live-bench","signalToNoise":{"available":true,"value":3.12},"difficulty":{"available":true,"value":120.7}},{"slug":"os-world","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"piqa","signalToNoise":{"available":true,"value":4.399},"difficulty":{"available":true,"value":66.2}},{"slug":"science-qa","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"superglue","signalToNoise":{"available":true,"value":2.368},"difficulty":{"available":true,"value":64.5}},{"slug":"the-agent-company","signalToNoise":{"available":true,"value":10.233},"difficulty":{"available":true,"value":145.2}},{"slug":"the-agent-company-full-completion","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"trivia-qa","signalToNoise":{"available":true,"value":3.069},"difficulty":{"available":true,"value":76.8}},{"slug":"videomme","signalToNoise":{"available":true,"value":4.978},"difficulty":{"available":true,"value":121.8}},{"slug":"apex-agents","signalToNoise":{"available":true,"value":9.575},"difficulty":{"available":true,"value":167.7}},{"slug":"arc-agi-2","signalToNoise":{"available":true,"value":14.162},"difficulty":{"available":true,"value":157.2}},{"slug":"hle","signalToNoise":{"available":true,"value":10.522},"difficulty":{"available":true,"value":163.5}},{"slug":"hle-with-tools","signalToNoise":{"available":true,"value":7.364},"difficulty":{"available":true,"value":156.9}},{"slug":"hle-text-only","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"charxiv-reasoning-with-tools-card","signalToNoise":{"available":true,"value":11.963},"difficulty":{"available":true,"value":131.3}},{"slug":"officeqa-pro-card","signalToNoise":{"available":true,"value":3.361},"difficulty":{"available":true,"value":154.9}},{"slug":"officeqa-pro-v2-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gdpval-aa-v1-card","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpval-aa-v2-card","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"healthbench-card","signalToNoise":{"available":true,"value":1.686},"difficulty":{"available":true,"value":138.6}},{"slug":"healthbench-card-hard","signalToNoise":{"available":true,"value":2.526},"difficulty":{"available":true,"value":167.7}},{"slug":"healthbench-card-hard-updated-implementation","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"healthbench-card-hard-length-adjusted","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"healthbench-card-updated-implementation","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"healthbench-card-length-adjusted","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"healthbench-card-length-adjusted-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"latchbio-singlecellbench-card","signalToNoise":{"available":true,"value":9.038},"difficulty":{"available":true,"value":155.4}},{"slug":"aikido-known-cve-detection","signalToNoise":{"available":true,"value":2.507},"difficulty":{"available":true,"value":135.5}},{"slug":"aikido-known-cve-detection-single-run-average","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aikido-known-cve-detection-32","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"aikido-known-cve-detection-32-single-run-average","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-4","signalToNoise":{"available":true,"value":5.044},"difficulty":{"available":true,"value":154.3}},{"slug":"eq-bench-4-bond-and-rapport","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eq-bench-4-authenticity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eq-bench-4-attunement","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eq-bench-4-meeting-preferences-and-needs","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eq-bench-4-emotion-sensemaking","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eq-bench-4-emotion-management","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eq-bench-3","signalToNoise":{"available":true,"value":3.411},"difficulty":{"available":true,"value":138.9}},{"slug":"eq-bench-3-rubric","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eq-bench-3-humanlike","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-safety","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-assertive","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-social-iq","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-warm","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-analytic","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-insight","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-empathy","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-compliant","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-moralising","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"eq-bench-3-pragmatic","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"scicode-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ifbench-artificial-analysis","signalToNoise":{"available":true,"value":2.285},"difficulty":{"available":true,"value":137.3}},{"slug":"aa-lcr-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-hard-artificial-analysis","signalToNoise":{"available":true,"value":8.923},"difficulty":{"available":true,"value":155.3}},{"slug":"math-500-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"banking-artificial-analysis","signalToNoise":{"available":true,"value":6.295},"difficulty":{"available":true,"value":167.8}},{"slug":"itbench-aa-artificial-analysis","signalToNoise":{"available":true,"value":4.501},"difficulty":{"available":true,"value":163.0}},{"slug":"enterpriseops-gym-aa-artificial-analysis","signalToNoise":{"available":true,"value":5.93},"difficulty":{"available":true,"value":165.0}},{"slug":"apex-agents-aa-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpval-aa-v2-normalized-artificial-analysis","signalToNoise":{"available":true,"value":9.01},"difficulty":{"available":true,"value":158.7}},{"slug":"aa-omniscience-index-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"epibench-benchmarks-bio","signalToNoise":{"available":true,"value":6.334},"difficulty":{"available":true,"value":170.7}},{"slug":"spatialbench-long-benchmarks-bio","signalToNoise":{"available":true,"value":10.96},"difficulty":{"available":true,"value":190.6}},{"slug":"txbench-pp-benchmarks-bio","signalToNoise":{"available":true,"value":28.435},"difficulty":{"available":true,"value":157.7}},{"slug":"bullshitbench-v2-leaderboard-bullshit-benchmark","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"deepswe-datacurve-2","signalToNoise":{"available":true,"value":16.66},"difficulty":{"available":true,"value":158.9}},{"slug":"deepswe-v1-pass-at-4","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deepswe-pass-4-datacurve-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitbench-leaderboard-mean-capability-exploitbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitbench-leaderboard-mean-capability-score-out-of-16-exploitbench","signalToNoise":{"available":true,"value":9.717},"difficulty":{"available":true,"value":163.6}},{"slug":"frontiercode-diamond-score-cognition-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercode-diamond-pass-rate-cognition-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercode-main-score-cognition-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercode-main-pass-rate-cognition-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercode-extended-score-cognition-2","signalToNoise":{"available":true,"value":14.737},"difficulty":{"available":true,"value":160.8}},{"slug":"frontiercode-extended-pass-rate-cognition-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gba-eval-overall-gbaeval","signalToNoise":{"available":true,"value":6.969},"difficulty":{"available":true,"value":161.8}},{"slug":"halluhard-all-domains-average-halluhard","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"lisanbench-path-length-lisanbench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"lisanbench-difficulty-weighted-lisanbench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"arxivlean-03-2026-matharena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"arxivlean-06-2026-matharena","signalToNoise":{"available":true,"value":38.498},"difficulty":{"available":true,"value":167.0}},{"slug":"arxivlean-09-2026-matharena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"brokenarxiv-overall-matharena","signalToNoise":{"available":true,"value":5.703},"difficulty":{"available":true,"value":168.2}},{"slug":"brokenarxiv-02-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"brokenarxiv-03-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"brokenarxiv-04-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"brokenarxiv-05-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"brokenarxiv-06-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-overall-matharena","signalToNoise":{"available":true,"value":10.843},"difficulty":{"available":true,"value":154.4}},{"slug":"arxivmath-12-2025-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-01-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-02-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-03-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-04-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-05-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-06-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"visual-math-overall-matharena","signalToNoise":{"available":true,"value":6.185},"difficulty":{"available":true,"value":133.2}},{"slug":"kangaroo-2025-1-2-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kangaroo-2025-3-4-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kangaroo-2025-5-6-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kangaroo-2025-7-8-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kangaroo-2025-9-10-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kangaroo-2025-11-12-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"final-answer-comps-overall-matharena","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hmmt-feb-2025-matharena","signalToNoise":{"available":true,"value":4.936},"difficulty":{"available":true,"value":136.1}},{"slug":"brumo-2025-matharena","signalToNoise":{"available":true,"value":9.494},"difficulty":{"available":true,"value":124.8}},{"slug":"smt-2025-matharena","signalToNoise":{"available":true,"value":9.909},"difficulty":{"available":true,"value":125.2}},{"slug":"cmimc-2025-matharena","signalToNoise":{"available":true,"value":9.933},"difficulty":{"available":true,"value":136.5}},{"slug":"hmmt-nov-2025-matharena","signalToNoise":{"available":true,"value":7.641},"difficulty":{"available":true,"value":126.5}},{"slug":"aime-2026-matharena","signalToNoise":{"available":true,"value":9.731},"difficulty":{"available":true,"value":127.2}},{"slug":"hmmt-feb-2026-matharena","signalToNoise":{"available":true,"value":11.401},"difficulty":{"available":true,"value":134.3}},{"slug":"apex-matharena-2","signalToNoise":{"available":true,"value":15.987},"difficulty":{"available":true,"value":159.3}},{"slug":"apex-shortlist-matharena","signalToNoise":{"available":true,"value":8.153},"difficulty":{"available":true,"value":146.7}},{"slug":"usamo-2025-matharena","signalToNoise":{"available":true,"value":5.514},"difficulty":{"available":true,"value":162.5}},{"slug":"imo-2025-matharena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"imc-2025-matharena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mikl-s-schweitzer-2025-matharena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"putnam-2025-matharena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"project-euler-matharena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"apex-mercor-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"apex-swe-mercor-2","signalToNoise":{"available":true,"value":7.642},"difficulty":{"available":true,"value":162.5}},{"slug":"opus-magnum-campaign-opus-magnum-bench-2","signalToNoise":{"available":true,"value":12.409},"difficulty":{"available":true,"value":163.2}},{"slug":"osworld-self-reported-a11y-tree-osworld","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-self-reported-screenshot-a11y-tree-osworld","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-self-reported-set-of-mark-osworld","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"posttrainbench-average-posttrainbench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"pencil-puzzle-bench-direct-ask-ppbench","signalToNoise":{"available":true,"value":10.532},"difficulty":{"available":true,"value":168.8}},{"slug":"pencil-puzzle-bench-agentic-ppbench","signalToNoise":{"available":true,"value":19.613},"difficulty":{"available":true,"value":157.6}},{"slug":"swe-atlas-refactoring-scale-ai-2","signalToNoise":{"available":true,"value":8.53},"difficulty":{"available":true,"value":162.2}},{"slug":"swe-atlas-test-writing-scale-ai-2","signalToNoise":{"available":true,"value":11.147},"difficulty":{"available":true,"value":162.0}},{"slug":"swe-atlas-codebase-qna-scale-ai-2","signalToNoise":{"available":true,"value":5.411},"difficulty":{"available":true,"value":163.7}},{"slug":"hil-bench-human-in-loop-benchmark-scale-ai-2","signalToNoise":{"available":true,"value":5.602},"difficulty":{"available":true,"value":162.0}},{"slug":"swe-bench-pro-public-dataset-scale-ai-2","signalToNoise":{"available":true,"value":7.174},"difficulty":{"available":true,"value":151.0}},{"slug":"swe-bench-pro-private-dataset-scale-ai-2","signalToNoise":{"available":true,"value":9.505},"difficulty":{"available":true,"value":160.9}},{"slug":"scipredict-scale-ai-2","signalToNoise":{"available":true,"value":3.342},"difficulty":{"available":true,"value":218.2}},{"slug":"humanity-s-last-exam-text-only-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"professional-reasoning-benchmark-finance-scale-ai-2","signalToNoise":{"available":true,"value":3.145},"difficulty":{"available":true,"value":159.4}},{"slug":"professional-reasoning-benchmark-legal-scale-ai-2","signalToNoise":{"available":true,"value":2.685},"difficulty":{"available":true,"value":161.1}},{"slug":"propensitybench-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"visualtoolbench-vtb-scale-ai-2","signalToNoise":{"available":true,"value":11.784},"difficulty":{"available":true,"value":164.5}},{"slug":"multinrc-scale-ai-2","signalToNoise":{"available":true,"value":8.581},"difficulty":{"available":true,"value":152.5}},{"slug":"multichallenge-scale-ai-2","signalToNoise":{"available":true,"value":4.274},"difficulty":{"available":true,"value":141.2}},{"slug":"fortress-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"mask-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"enigmaeval-scale-ai-2","signalToNoise":{"available":true,"value":24.379},"difficulty":{"available":true,"value":166.6}},{"slug":"tutorbench-scale-ai-2","signalToNoise":{"available":true,"value":3.223},"difficulty":{"available":true,"value":144.3}},{"slug":"adversarial-robustness-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"agentic-tool-use-chat-scale-ai-2","signalToNoise":{"available":true,"value":4.324},"difficulty":{"available":true,"value":130.1}},{"slug":"agentic-tool-use-enterprise-scale-ai-2","signalToNoise":{"available":true,"value":5.685},"difficulty":{"available":true,"value":124.9}},{"slug":"humanity-s-last-exam-preview-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"humanity-s-last-exam-text-only-preview-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"instruction-following-scale-ai-2","signalToNoise":{"available":true,"value":6.399},"difficulty":{"available":true,"value":77.6}},{"slug":"arabic-scale-ai-2","signalToNoise":{"available":true,"value":1.751},"difficulty":{"available":true,"value":128.2}},{"slug":"chinese-scale-ai-2","signalToNoise":{"available":true,"value":3.182},"difficulty":{"available":true,"value":128.2}},{"slug":"coding-scale-ai-2","signalToNoise":{"available":true,"value":4.778},"difficulty":{"available":true,"value":126.2}},{"slug":"japanese-scale-ai-2","signalToNoise":{"available":true,"value":3.022},"difficulty":{"available":true,"value":127.5}},{"slug":"korean-scale-ai-2","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"spanish-scale-ai-2","signalToNoise":{"available":true,"value":3.22},"difficulty":{"available":true,"value":127.9}},{"slug":"slopcodebench-ckpt-solved-scbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"slopcodebench-isolated-solved-scbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"slopcodebench-core-solved-scbench","signalToNoise":{"available":true,"value":10.676},"difficulty":{"available":true,"value":153.6}},{"slug":"slopcodebench-problems-solved-scbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"slopcodebench-erosion-mean-scbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"slopcodebench-verbosity-mean-scbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"slopcodebench-ast-grep-scbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"slopcodebench-cloned-scbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"complexconstraints-surge-ai-2","signalToNoise":{"available":true,"value":7.153},"difficulty":{"available":true,"value":165.4}},{"slug":"handbook-md-surge-ai-2","signalToNoise":{"available":true,"value":16.371},"difficulty":{"available":true,"value":170.4}},{"slug":"antidote-everyday-edition-surge-ai-2","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"enterprisebench-corecraft-surge-ai-2","signalToNoise":{"available":true,"value":6.211},"difficulty":{"available":true,"value":160.7}},{"slug":"hemingway-bench-surge-ai-2","signalToNoise":{"available":true,"value":4.805},"difficulty":{"available":true,"value":154.2}},{"slug":"automationbench-overall-zapier-benchmarks","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"automationbench-sales-zapier-benchmarks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-marketing-zapier-benchmarks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-operations-zapier-benchmarks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-support-zapier-benchmarks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-finance-zapier-benchmarks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-hr-zapier-benchmarks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-public-set","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"zerobench-main-questions-pass-1-zerobench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"zerobench-subquestions-pass-1-zerobench","signalToNoise":{"available":true,"value":3.992},"difficulty":{"available":true,"value":172.2}},{"slug":"finance-agent-v2","signalToNoise":{"available":true,"value":7.508},"difficulty":{"available":true,"value":157.4}},{"slug":"livecodebench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro","signalToNoise":{"available":true,"value":6.89},"difficulty":{"available":true,"value":116.7}},{"slug":"mmlu-pro-vals-5-shot-cot","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmmu-pro","signalToNoise":{"available":true,"value":6.864},"difficulty":{"available":true,"value":132.4}},{"slug":"mmmu-pro-vals-4-option","signalToNoise":{"available":true,"value":3.535},"difficulty":{"available":true,"value":128.7}},{"slug":"terminal-bench-2-1","signalToNoise":{"available":true,"value":8.686},"difficulty":{"available":true,"value":148.7}},{"slug":"arc-agi-1","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"cursorbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cursorbench-3-2","signalToNoise":{"available":true,"value":15.673},"difficulty":{"available":true,"value":154.9}},{"slug":"riemannbench","signalToNoise":{"available":true,"value":9.942},"difficulty":{"available":true,"value":164.8}},{"slug":"riemannbench-anthropic-no-tools","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"riemannbench-anthropic-corrected-grader-tools","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"usamo-2026","signalToNoise":{"available":true,"value":13.492},"difficulty":{"available":true,"value":151.5}},{"slug":"critpt","signalToNoise":{"available":true,"value":15.298},"difficulty":{"available":true,"value":171.1}},{"slug":"critpt-v2","signalToNoise":{"available":true,"value":11.515},"difficulty":{"available":true,"value":159.1}},{"slug":"blueprint-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"blueprint-bench-2","signalToNoise":{"available":true,"value":8.975},"difficulty":{"available":true,"value":166.9}},{"slug":"butter-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"butter-bench-search-package","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"butter-bench-infer-butter-bag","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"butter-bench-notice-absence","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"butter-bench-wait-confirmed-pickup","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"butter-bench-multi-step-spatial-planning","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"butter-bench-e2e-pass-the-butter","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"osworld-verified","signalToNoise":{"available":true,"value":6.718},"difficulty":{"available":true,"value":147.2}},{"slug":"osworld-verified-anthropic-361","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-verified-no-google-drive-361","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ramp-swe-bench-observed-78","signalToNoise":{"available":true,"value":12.338},"difficulty":{"available":true,"value":143.6}},{"slug":"simple-qa-verified-f1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitgym-v0","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-v0-userspace","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-v0-v8","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-v0-kernel","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-v0-with-mitigations","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-v0-userspace-with-mitigations","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-v0-v8-with-mitigations","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-v0-kernel-with-mitigations","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mcp-atlas","signalToNoise":{"available":true,"value":4.285},"difficulty":{"available":true,"value":147.8}},{"slug":"vending-bench-2","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-1","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-12","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-2","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-3","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-4","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-5-opus-4-6","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-5-sonnet-4-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-6","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-7","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-8","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-9","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-arena-round-10","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"aime-2025","signalToNoise":{"available":true,"value":5.335},"difficulty":{"available":true,"value":131.6}},{"slug":"aime-2025-average-10-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"remote-labor-index","signalToNoise":{"available":true,"value":27.204},"difficulty":{"available":true,"value":175.9}},{"slug":"mmmu","signalToNoise":{"available":true,"value":9.023},"difficulty":{"available":true,"value":122.6}},{"slug":"mgsm","signalToNoise":{"available":true,"value":4.964},"difficulty":{"available":true,"value":101.5}},{"slug":"aime-2024","signalToNoise":{"available":true,"value":3.197},"difficulty":{"available":true,"value":131.7}},{"slug":"ifeval","signalToNoise":{"available":true,"value":2.692},"difficulty":{"available":true,"value":90.3}},{"slug":"arc-challenge","signalToNoise":{"available":true,"value":5.14},"difficulty":{"available":true,"value":93.6}},{"slug":"mmmlu","signalToNoise":{"available":true,"value":5.297},"difficulty":{"available":true,"value":118.4}},{"slug":"tau2-bench-telecom","signalToNoise":{"available":true,"value":1.952},"difficulty":{"available":true,"value":133.8}},{"slug":"browsecomp","signalToNoise":{"available":true,"value":8.333},"difficulty":{"available":true,"value":146.8}},{"slug":"cybergym","signalToNoise":{"available":true,"value":9.995},"difficulty":{"available":true,"value":151.4}},{"slug":"charxiv-reasoning","signalToNoise":{"available":true,"value":4.297},"difficulty":{"available":true,"value":128.2}},{"slug":"simpleqa","signalToNoise":{"available":true,"value":2.927},"difficulty":{"available":true,"value":153.0}},{"slug":"arc-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"big-bench-extra-hard","signalToNoise":{"available":true,"value":20.705},"difficulty":{"available":true,"value":140.6}},{"slug":"math-vision","signalToNoise":{"available":true,"value":6.823},"difficulty":{"available":true,"value":134.3}},{"slug":"vals-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vals-multimodal-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vals-science-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vals-finance-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vals-legal-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"eci","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"aa-intelligence-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"aa-coding-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"aa-agentic-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-arena-elo","signalToNoise":{"available":true,"value":4.224},"difficulty":{"available":true,"value":136.0}},{"slug":"bfcl-v4-overall-accuracy","signalToNoise":{"available":true,"value":2.519},"difficulty":{"available":true,"value":136.0}},{"slug":"spiral-bench-v1-2-safety-score","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"longform-creative-writing","signalToNoise":{"available":true,"value":3.824},"difficulty":{"available":true,"value":137.2}},{"slug":"creative-writing-v3-elo","signalToNoise":{"available":true,"value":3.38},"difficulty":{"available":true,"value":142.8}},{"slug":"creative-writing-v3-rubric","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"slop-score","signalToNoise":{"available":true,"value":1.622},"difficulty":{"available":true,"value":133.1}},{"slug":"judgemark-v4","signalToNoise":{"available":true,"value":3.816},"difficulty":{"available":true,"value":148.3}},{"slug":"buzzbench","signalToNoise":{"available":true,"value":5.596},"difficulty":{"available":true,"value":126.6}},{"slug":"diplobench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"spiral-bench-v1-0-safety-score","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"creative-writing-v2","signalToNoise":{"available":true,"value":3.147},"difficulty":{"available":true,"value":91.0}},{"slug":"eq-bench-v2","signalToNoise":{"available":true,"value":3.82},"difficulty":{"available":true,"value":91.0}},{"slug":"magi-hard","signalToNoise":{"available":true,"value":9.82},"difficulty":{"available":true,"value":115.6}},{"slug":"eq-bench-v2-magi-hard-combined","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"judgemark-v2-1","signalToNoise":{"available":true,"value":5.381},"difficulty":{"available":true,"value":129.6}},{"slug":"swe-rebench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"swe-rebench-pass-5","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"swe-marathon","signalToNoise":{"available":true,"value":11.308},"difficulty":{"available":true,"value":171.6}},{"slug":"mathvista-mini","signalToNoise":{"available":true,"value":3.985},"difficulty":{"available":true,"value":116.0}},{"slug":"deepsearchqa-f1-score","signalToNoise":{"available":true,"value":7.328},"difficulty":{"available":true,"value":144.2}},{"slug":"kimi-code-bench-v2","signalToNoise":{"available":true,"value":6.096},"difficulty":{"available":true,"value":144.4}},{"slug":"mls-bench-lite","signalToNoise":{"available":true,"value":6.177},"difficulty":{"available":true,"value":168.3}},{"slug":"erqa","signalToNoise":{"available":true,"value":6.039},"difficulty":{"available":true,"value":153.7}},{"slug":"forecastbench-tournament-overall-brier-index","signalToNoise":{"available":true,"value":4.584},"difficulty":{"available":true,"value":209.7}},{"slug":"forecastbench-tournament-dataset-brier-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"forecastbench-tournament-market-brier-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"forecastbench-baseline-overall-brier-index","signalToNoise":{"available":true,"value":5.604},"difficulty":{"available":true,"value":218.6}},{"slug":"forecastbench-baseline-dataset-brier-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"forecastbench-baseline-market-brier-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"forecastbench-preliminary-dataset-brier-index","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"confabulations-leaderboard-confabulation-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"confabulations-leaderboard-non-response-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"confabulations-leaderboard-weighted-score","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"genebench-pro","signalToNoise":{"available":true,"value":29.699},"difficulty":{"available":true,"value":167.8}},{"slug":"genebench-pro-public-release-subset","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"genebench-pro-artificial-analysis-subset","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"genebench-pro-externally-reviewed","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"genebench-pro-not-externally-reviewed","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"bfcl","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vending-bench-units-sold","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-net-worth-min","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-units-sold-min","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-days-until-sales-stop","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"vending-bench-days-until-sales-stop-percentage","signalToNoise":{"available":true,"value":2.845},"difficulty":{"available":true,"value":128.8}},{"slug":"agents-last-exam-ale","signalToNoise":{"available":true,"value":13.413},"difficulty":{"available":true,"value":172.2}},{"slug":"continual-learning-bench-cl-bench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"olmocr-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"omnidocbench-v1-5","signalToNoise":{"available":true,"value":2.379},"difficulty":{"available":true,"value":79.3}},{"slug":"snorkel-underwrite","signalToNoise":{"available":true,"value":2.647},"difficulty":{"available":true,"value":130.6}},{"slug":"phare","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-average-safety","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-hallucination-resistance","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-harm-resistance","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-bias-resistance","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-jailbreak-resistance","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-factuality","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-factuality-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-factuality-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-factuality-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-misinformation","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-misinformation-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-misinformation-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-misinformation-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-debunking","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-debunking-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-debunking-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-debunking-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-plugin-reliability","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-plugin-reliability-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-plugin-reliability-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-plugin-reliability-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-harmful-misguidance","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-harmful-misguidance-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-harmful-misguidance-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-harmful-misguidance-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-encoding-jailbreaks","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-encoding-jailbreaks-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-encoding-jailbreaks-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-encoding-jailbreaks-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-framing-jailbreaks","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-framing-jailbreaks-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-framing-jailbreaks-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-framing-jailbreaks-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-prompt-injection","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-prompt-injection-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-prompt-injection-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-prompt-injection-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-self-assessed-stereotypes","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-self-assessed-stereotypes-en","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-self-assessed-stereotypes-es","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"phare-self-assessed-stereotypes-fr","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"math-500","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arena-ai-document","signalToNoise":{"available":true,"value":4.156},"difficulty":{"available":true,"value":156.0}},{"slug":"blxbench","signalToNoise":{"available":true,"value":1.312},"difficulty":{"available":true,"value":103.4}},{"slug":"caselaw-v2","signalToNoise":{"available":true,"value":1.089},"difficulty":{"available":true,"value":116.8}},{"slug":"llm-debate-benchmark","signalToNoise":{"available":true,"value":7.241},"difficulty":{"available":true,"value":154.0}},{"slug":"lmarena-search-arena","signalToNoise":{"available":true,"value":1.686},"difficulty":{"available":true,"value":158.3}},{"slug":"lmarena-vision-arena","signalToNoise":{"available":true,"value":4.534},"difficulty":{"available":true,"value":145.2}},{"slug":"lmarena-vision-arena-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"thematic-generalization-v2","signalToNoise":{"available":true,"value":8.592},"difficulty":{"available":true,"value":140.0}},{"slug":"zerobench-main-questions-pass-5-zerobench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"zerobench-main-questions-pass-caret-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-omniscience-accuracy-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aa-omniscience-hallucination-rate-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"swe-bench-lite","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"lmarena-text-arena-coding-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-math-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-instruction-following-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-creative-writing-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-multiturn-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-hard-prompts-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-hard-prompts-english-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-longer-query-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-verified-vals-1-4-hours","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-verified-vals-15-min-1-hour","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"buyout-game-bradley-terry","signalToNoise":{"available":true,"value":4.821},"difficulty":{"available":true,"value":149.1}},{"slug":"extended-nyt-connections-940","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"extended-nyt-connections-newest-100","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livebench-agentic-coding-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-coding-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-data-analysis-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-instruction-following-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-language-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-mathematics-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-reasoning-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"llm-creative-story-writing","signalToNoise":{"available":true,"value":2.64},"difficulty":{"available":true,"value":156.0}},{"slug":"llm-position-bias-decisive-coverage","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"llm-position-bias-first-shown-pick","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"llm-position-bias-order-flip","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"llm-position-bias-rating-bonus","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"pact-negotiation-cms","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-expert-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mbpp-plus","signalToNoise":{"available":true,"value":1.715},"difficulty":{"available":true,"value":89.4}},{"slug":"mindcube-tiny","signalToNoise":{"available":true,"value":7.498},"difficulty":{"available":true,"value":152.2}},{"slug":"spatialviz-bench","signalToNoise":{"available":true,"value":12.382},"difficulty":{"available":true,"value":155.0}},{"slug":"lmarena-text-arena-industry-writing-and-literature-and-language-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"apex-v1-extended","signalToNoise":{"available":true,"value":4.021},"difficulty":{"available":true,"value":141.7}},{"slug":"medqa","signalToNoise":{"available":true,"value":3.782},"difficulty":{"available":true,"value":111.1}},{"slug":"lmarena-text-arena-english-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-software-it-services-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-business-management-financial-ops-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-legal-government-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-medicine-healthcare-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-mathematical-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-agent-arena-net-improvement","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-confirmed-success","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-praise-vs-complaint","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-steerability","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-bash-recovery","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-tool-hallucination","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-briefcase","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-openness-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-arena-chinese-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-japanese-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-coding-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-2-0","signalToNoise":{"available":true,"value":9.88},"difficulty":{"available":true,"value":150.5}},{"slug":"lmarena-vision-ocr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"taxbench-total-pass1","signalToNoise":{"available":true,"value":3.22},"difficulty":{"available":true,"value":130.4}},{"slug":"taxbench-total-pass5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"taxbench-knowledge-pass1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"taxbench-knowledge-pass5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"taxbench-calculations-pass1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"taxbench-calculations-pass5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"taxbench-retrieval-pass1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"taxbench-retrieval-pass5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"babyvision","signalToNoise":{"available":true,"value":2.823},"difficulty":{"available":true,"value":160.6}},{"slug":"bird-critic","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"llm-position-bias-first-lift","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"llm-sycophancy-conditional-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-conditional-total-inconsistency","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-contrarian-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-decisive-coverage","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-insufficient-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-stripped-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-total-inconsistency","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"clockbench-accuracy","signalToNoise":{"available":true,"value":4.306},"difficulty":{"available":true,"value":164.5}},{"slug":"prinzbench","signalToNoise":{"available":true,"value":4.854},"difficulty":{"available":true,"value":159.4}},{"slug":"prinzbench-legal-research","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"prinzbench-search","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"babyvision-with-python","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"math-vision-with-python","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"biotier-refusal","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"textquests-average-cumulative-harm","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"textquests-game-progress","signalToNoise":{"available":true,"value":7.774},"difficulty":{"available":true,"value":162.9}},{"slug":"gdp-pdf","signalToNoise":{"available":true,"value":12.018},"difficulty":{"available":true,"value":176.1}},{"slug":"code-migration-code-quality","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agieval-en","signalToNoise":{"available":true,"value":11.835},"difficulty":{"available":true,"value":113.9}},{"slug":"agieval-zh","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"mgsm-bengali","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-chinese","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-english","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-french","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-german","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-japanese","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-russian","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-spanish","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-swahili","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-telugu","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mgsm-thai","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-math-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-overall-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"charxiv-descriptive","signalToNoise":{"available":true,"value":6.528},"difficulty":{"available":true,"value":100.7}},{"slug":"code-migration-cli","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"code-migration-cobol","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"corp_fin_v2-exact-pages","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"corp_fin_v2-max-fitting-context","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"corp_fin_v2-shared-max-context","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-adjustments","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-all-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-comparables","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-disclosure-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-earnings-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-financial-modeling","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-general-qualitative-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-general-quantitative-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-market-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-precedents","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiermath-tiers-1-3-v1-private","signalToNoise":{"available":true,"value":12.016},"difficulty":{"available":true,"value":162.0}},{"slug":"frontiermath-tier-4-v1-private","signalToNoise":{"available":true,"value":14.919},"difficulty":{"available":true,"value":167.8}},{"slug":"frontiermath-tier-4-v2-private","signalToNoise":{"available":true,"value":9.043},"difficulty":{"available":true,"value":160.7}},{"slug":"frontiermath-tiers-1-3-v2-private","signalToNoise":{"available":true,"value":9.904},"difficulty":{"available":true,"value":152.8}},{"slug":"hlab-antitrust-competition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-arbitration-international-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-banking-finance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-bankruptcy-restructuring","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-capital-markets","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-corporate-governance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-corporate-ma","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-antitrust-competition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-arbitration-international-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-banking-finance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-bankruptcy-restructuring","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-capital-markets","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-corporate-governance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-corporate-ma","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-data-privacy-cybersecurity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-emerging-companies-venture-capital","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-employment-labor","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-energy-natural-resources","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-environmental-esg","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-funds-asset-management","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-healthcare-life-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-immigration","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-insurance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-intellectual-property","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-international-trade-sanctions","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-litigation-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-real-estate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-structured-finance-securitization","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-tax","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-trusts-estates-private-client","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-criteria-pass-rate-white-collar-defense-investigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-data-privacy-cybersecurity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-emerging-companies-venture-capital","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-employment-labor","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-energy-natural-resources","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-environmental-esg","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-funds-asset-management","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-healthcare-life-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-immigration","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-insurance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-intellectual-property","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-international-trade-sanctions","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-litigation-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-real-estate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-structured-finance-securitization","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-tax","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-trusts-estates-private-client","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-white-collar-defense-investigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ioi-ioi2024","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ioi-ioi2025","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_bench-conclusion-tasks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_bench-interpretation-tasks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_bench-issue-tasks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_bench-rhetoric-tasks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_bench-rule-tasks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-biology","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-business","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-chemistry","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-computer-science","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-economics","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-engineering","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-health","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-history","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-law","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-math","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-other","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-philosophy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-physics","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmlu-pro-psychology","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mortgage_tax-numerical-extraction","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"mortgage_tax-semantic-extraction","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"mortgage_tax-numerical-extraction-vals-all-1258","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mortgage_tax-semantic-extraction-vals-all-1258","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"programbench-counts-almost","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"programbench-counts-strict","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"public-benefits-bench-both","signalToNoise":{"available":true,"value":4.813},"difficulty":{"available":true,"value":145.7}},{"slug":"public-benefits-bench-multi-turn","signalToNoise":{"available":true,"value":9.402},"difficulty":{"available":true,"value":153.9}},{"slug":"public-benefits-bench-neither","signalToNoise":{"available":true,"value":8.725},"difficulty":{"available":true,"value":159.3}},{"slug":"public-benefits-bench-web-search","signalToNoise":{"available":true,"value":6.306},"difficulty":{"available":true,"value":154.1}},{"slug":"sage-automata","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sage-calculus","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sage-linear-algebra","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sage-pdes","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-verified-15-min-fix","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-verified-4-hours","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax_eval_v2-correctness","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax_eval_v2-stepwise-reasoning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-2-1-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-2-1-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-2-1-medium","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"global-mmlu-lite","signalToNoise":{"available":true,"value":6.683},"difficulty":{"available":true,"value":118.9}},{"slug":"global-mmlu-lite-inkling-unspecified-languages","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"global-mmlu-lite-google-15-languages","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"global-mmlu-lite-google-unspecified-languages","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"llm-persuasion-average-strength","signalToNoise":{"available":true,"value":5.012},"difficulty":{"available":true,"value":128.5}},{"slug":"llm-persuasion-target-susceptibility","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"lmarena-text-arena-german-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-korean-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-entertainment-sports-media-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-life-physical-social-science-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-russian-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-spanish-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"pact-negotiation-average-profit-per-round","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"thematic-generalization-v2-inverse-rank-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"scale-seal-visual-language-understanding","signalToNoise":{"available":true,"value":3.811},"difficulty":{"available":true,"value":150.4}},{"slug":"frontierscience-olympiad","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"aidanbench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"zeroeval","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"medqa-vals-demographic-bias-unbiased","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"medqa-vals-demographic-bias-hispanic","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"medqa-vals-demographic-bias-black","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"medqa-vals-demographic-bias-asian","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"medqa-vals-demographic-bias-white","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"medqa-vals-demographic-bias-indigenous","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-2-0-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-2-0-medium","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-2-0-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aime-2024-2025","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snakebench-average-apples","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snakebench-best-apples","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snakebench-rating","signalToNoise":{"available":true,"value":1.367},"difficulty":{"available":true,"value":134.9}},{"slug":"snakebench-total-apples","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snakebench-win-rate","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"refcoco","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"winogrande","signalToNoise":{"available":true,"value":3.984},"difficulty":{"available":true,"value":102.4}},{"slug":"ocrbench-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"openbookqa","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openbookqa-helm-lite","signalToNoise":{"available":true,"value":6.925},"difficulty":{"available":true,"value":85.3}},{"slug":"ramp-swe-bench","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"voratiq-coding-agent-leaderboard","signalToNoise":{"available":true,"value":4.616},"difficulty":{"available":true,"value":156.9}},{"slug":"voxelbench-text-rating","signalToNoise":{"available":true,"value":8.776},"difficulty":{"available":true,"value":159.1}},{"slug":"voxelbench-image-rating","signalToNoise":{"available":true,"value":3.86},"difficulty":{"available":true,"value":156.2}},{"slug":"pact-negotiation-bilateral-rating","signalToNoise":{"available":true,"value":4.005},"difficulty":{"available":true,"value":148.4}},{"slug":"step-game-trueskill-mu","signalToNoise":{"available":true,"value":3.83},"difficulty":{"available":true,"value":137.7}},{"slug":"elimination-game-trueskill-mu","signalToNoise":{"available":true,"value":2.174},"difficulty":{"available":true,"value":137.9}},{"slug":"pgg-bench-trueskill-mu","signalToNoise":{"available":true,"value":1.374},"difficulty":{"available":true,"value":134.4}},{"slug":"public-goods-game-average-contribution","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"llm-deception-disinformation-effectiveness","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-deception-vulnerability-score","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bazaar-trueskill-mu","signalToNoise":{"available":true,"value":4.061},"difficulty":{"available":true,"value":134.8}},{"slug":"bazaar-illegality-high-score-games","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-divergent-thinking-score","signalToNoise":{"available":true,"value":2.772},"difficulty":{"available":true,"value":152.2}},{"slug":"llm-divergent-thinking-repeat-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"round-trip-translation-mean-score","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"llm-creative-story-writing-win-probability","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-coding","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-cost","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-debugging","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-hallucination","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-reasoning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-refactoring","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-security","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-speed","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blxbench-category-ui","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"skillsbench-with-skills","signalToNoise":{"available":true,"value":5.529},"difficulty":{"available":true,"value":155.8}},{"slug":"skillsbench-without-skills","signalToNoise":{"available":true,"value":8.545},"difficulty":{"available":true,"value":163.4}},{"slug":"solo-bench-easy","signalToNoise":{"available":true,"value":8.664},"difficulty":{"available":true,"value":145.8}},{"slug":"solo-bench-medium","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"vending-bench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-attack","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-defence","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-strength","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-hitpoints","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-ranged","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-prayer","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-magic","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-woodcutting","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-fishing","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-mining","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-cooking","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-fletching","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-crafting","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-smithing","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-firemaking","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-skill-thieving","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-vanilla-15m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-smith-alch-15m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-fish-15m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-fletch-alch-15m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-vanilla-30m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-fish-30m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-smith-alch-30m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"runebench-gold-fletch-alch-30m","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"cyberbench-overall","signalToNoise":{"available":true,"value":1.451},"difficulty":{"available":true,"value":132.8}},{"slug":"cyberbench-poc","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cyberbench-patch","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hardgeobench-within-10-miles","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hardgeobench-within-100-miles","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hardgeobench-country-accuracy","signalToNoise":{"available":true,"value":0.891},"difficulty":{"available":true,"value":153.5}},{"slug":"hardgeobench-avg-distance-miles","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"algotune","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"ale-bench","signalToNoise":{"available":true,"value":3.982},"difficulty":{"available":true,"value":151.0}},{"slug":"cl-bench","signalToNoise":{"available":true,"value":6.207},"difficulty":{"available":true,"value":188.6}},{"slug":"cl-bench-life","signalToNoise":{"available":true,"value":13.964},"difficulty":{"available":true,"value":177.4}},{"slug":"vct-refusal-cais","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"machiavelli-deception-cais","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"intphys-2","signalToNoise":{"available":true,"value":15.289},"difficulty":{"available":true,"value":171.4}},{"slug":"art-agent-red-teaming-k100-cais","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"hle-rms-calibration-error-cais","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb","signalToNoise":{"available":true,"value":8.023},"difficulty":{"available":true,"value":155.3}},{"slug":"emb-template","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-scratch","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-lbo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-dcf","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-m-a","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-operating","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-comps","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-dataroom","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-misc","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research","signalToNoise":{"available":true,"value":9.34},"difficulty":{"available":true,"value":164.1}},{"slug":"legal_research-weighted-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-administrative-regulatory","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-business-commercial","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-civil-litigation","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-constitutional-civil-rights","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-criminal","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-family","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-health","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-immigration","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-index-finance-agent-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-index-corpfin-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-index-swe-bench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-index-vibe-code-bench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-index-terminal-bench-2-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-multimodal-index-finance-agent-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-multimodal-index-corpfin-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-multimodal-index-swe-bench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-multimodal-index-sage","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-multimodal-index-mortgage-tax","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-multimodal-index-vibe-code-bench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-multimodal-index-terminal-bench-2-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-administrative-regulatory","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-business-commercial","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-civil-litigation","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-constitutional-civil-rights","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-criminal","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-family","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-health","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"legal_research-weighted-pass-immigration","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"snorkelspatial","signalToNoise":{"available":true,"value":2.085},"difficulty":{"available":true,"value":147.3}},{"slug":"snorkel-finance","signalToNoise":{"available":true,"value":5.558},"difficulty":{"available":true,"value":133.1}},{"slug":"snorkel-sequences","signalToNoise":{"available":true,"value":2.055},"difficulty":{"available":true,"value":130.0}},{"slug":"snorkel-graph","signalToNoise":{"available":true,"value":5.053},"difficulty":{"available":true,"value":135.4}},{"slug":"snorkel-wordle","signalToNoise":{"available":true,"value":4.63},"difficulty":{"available":true,"value":128.8}},{"slug":"senior-swe-bench","signalToNoise":{"available":true,"value":7.91},"difficulty":{"available":true,"value":171.7}},{"slug":"hieroglyphbench","signalToNoise":{"available":true,"value":2.178},"difficulty":{"available":true,"value":179.9}},{"slug":"ebr-bench","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"ebr-bench-v1","signalToNoise":{"available":true,"value":4.757},"difficulty":{"available":true,"value":185.5}},{"slug":"ebr-bench-v2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"automationbench-aa-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-300-step-partial-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-150-step-partial-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livecodebench-artificial-analysis","signalToNoise":{"available":true,"value":4.789},"difficulty":{"available":true,"value":132.9}},{"slug":"livecodebench-vals-public","signalToNoise":{"available":true,"value":5.518},"difficulty":{"available":true,"value":129.6}},{"slug":"osworld-2-500-step-binary-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-500-step-partial-score","signalToNoise":{"available":true,"value":14.428},"difficulty":{"available":true,"value":161.1}},{"slug":"exploitgym","signalToNoise":{"available":true,"value":12.585},"difficulty":{"available":true,"value":172.6}},{"slug":"harvey-lab-aa-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"harvey-lab-aa-v1-1-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercode-1-1-main-score-cognition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercode-1-1-main-pass-rate-cognition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercode-1-1-extended-score-cognition","signalToNoise":{"available":true,"value":10.834},"difficulty":{"available":true,"value":158.7}},{"slug":"frontiercode-1-1-extended-pass-rate-cognition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gso","signalToNoise":{"available":true,"value":14.123},"difficulty":{"available":true,"value":159.6}},{"slug":"gso-hack-adjusted","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gso-opt-at-10","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gso-opt-at-10-hack-adjusted","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"scicode","signalToNoise":{"available":true,"value":7.731},"difficulty":{"available":true,"value":155.0}},{"slug":"sldbench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"sldbench-task-mean-r2","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"surface-evolver-bench","signalToNoise":{"available":true,"value":4.463},"difficulty":{"available":true,"value":151.3}},{"slug":"surface-evolver-bench-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deepswe-1-1","signalToNoise":{"available":true,"value":10.979},"difficulty":{"available":true,"value":159.1}},{"slug":"toolathlon","signalToNoise":{"available":true,"value":7.496},"difficulty":{"available":true,"value":156.8}},{"slug":"toolathlon-verified","signalToNoise":{"available":true,"value":13.335},"difficulty":{"available":true,"value":151.1}},{"slug":"babyvision-with-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arc-agi-3","signalToNoise":{"available":true,"value":6.896},"difficulty":{"available":true,"value":171.4}},{"slug":"samaya-frontierfinance-v3-1","signalToNoise":{"available":true,"value":4.097},"difficulty":{"available":true,"value":165.5}},{"slug":"samaya-frontierfinance-v3-1-financial-data-and-modeling","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-sector-industry-and-macro","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-earnings-and-events","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-company-research","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-coverage-and-catalyst-monitoring","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-screening-and-discovery","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-rubric-factual-data-extraction","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-rubric-qualitative-and-contextual-information","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-rubric-analysis-and-interpretation","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-rubric-comparative-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-rubric-forward-looking-information","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"samaya-frontierfinance-v3-1-rubric-format-and-presentation","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"jobbench-main","signalToNoise":{"available":true,"value":6.668},"difficulty":{"available":true,"value":160.3}},{"slug":"scbench-long-benchmarks-bio","signalToNoise":{"available":true,"value":17.856},"difficulty":{"available":true,"value":168.0}},{"slug":"biosecbench-refusal-benchmarks-bio","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"biosecbench-surveillance-benchmarks-bio","signalToNoise":{"available":true,"value":10.784},"difficulty":{"available":true,"value":159.6}},{"slug":"gbench-one-shot-coding-gscore","signalToNoise":{"available":true,"value":9.479},"difficulty":{"available":true,"value":158.2}},{"slug":"btf3-pooled","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"btf3-binary","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"btf3-numeric","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"policybench-us-exact-match","signalToNoise":{"available":true,"value":3.163},"difficulty":{"available":true,"value":114.6}},{"slug":"visual-acuity-4k-threshold-row","signalToNoise":{"available":true,"value":0.929},"difficulty":{"available":true,"value":119.9}},{"slug":"visual-acuity-1k-threshold-row","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"simplebench-open-ended","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"radle-2","signalToNoise":{"available":true,"value":2.761},"difficulty":{"available":true,"value":187.7}},{"slug":"radle-2-reliability-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"radle-2-accuracy-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"radle-2-safety-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"radle-2-handover-readiness-index","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aime-2025-average-8","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aime-2024-average-8","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hellaswag-unspecified-scoring-protocol","signalToNoise":{"available":true,"value":2.794},"difficulty":{"available":true,"value":74.6}},{"slug":"swe-bench-verified-anthropic-489","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-verified-epoch-484","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"zeroeval-gsm8k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"zeroeval-mmlu-redux","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"zeroeval-zebralogic","signalToNoise":{"available":true,"value":6.244},"difficulty":{"available":true,"value":132.4}},{"slug":"zeroeval-crux","signalToNoise":{"available":true,"value":9.135},"difficulty":{"available":true,"value":111.5}},{"slug":"zeroeval-math-l5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmmu-pro-standard-vision-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-150-step-binary-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-300-step-binary-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmmu-pro-with-python","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"voxpopuli-cleaned-aa","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"structured-output-benchmark-sob-value-accuracy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"spider-2-lite-sqlite-subset","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"medqa-vals-demographic-bias","signalToNoise":{"available":true,"value":5.325},"difficulty":{"available":true,"value":113.2}},{"slug":"livecodebench-vals-public-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livecodebench-vals-public-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livecodebench-vals-public-medium","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gsm1k","signalToNoise":{"available":true,"value":17.068},"difficulty":{"available":true,"value":96.4}},{"slug":"reactbench-v1","signalToNoise":{"available":true,"value":10.404},"difficulty":{"available":true,"value":165.4}},{"slug":"reactbench-v1-write-react","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"reactbench-v1-fix-react","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"audio-multichallenge-text-output","signalToNoise":{"available":true,"value":3.554},"difficulty":{"available":true,"value":154.7}},{"slug":"spreadsheetbench-2","signalToNoise":{"available":true,"value":9.064},"difficulty":{"available":true,"value":170.4}},{"slug":"deck-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"perceptionbench","signalToNoise":{"available":true,"value":3.431},"difficulty":{"available":true,"value":156.4}},{"slug":"zerobench-main-questions-pass-5-zerobench-with-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"variantbench-benchmarks-bio","signalToNoise":{"available":true,"value":10.565},"difficulty":{"available":true,"value":167.9}},{"slug":"spatialbench-benchmarks-bio","signalToNoise":{"available":true,"value":9.427},"difficulty":{"available":true,"value":156.2}},{"slug":"chartography-surge-ai-2","signalToNoise":{"available":true,"value":7.912},"difficulty":{"available":true,"value":166.7}},{"slug":"drugdiscoverybench","signalToNoise":{"available":true,"value":9.55},"difficulty":{"available":true,"value":160.9}},{"slug":"audio-multichallenge-audio-output","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gpqa-diamond-vals-protocol-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"snorkel-agentic-coding","signalToNoise":{"available":true,"value":6.777},"difficulty":{"available":true,"value":151.3}},{"slug":"snorkel-finance-reasoning","signalToNoise":{"available":true,"value":3.998},"difficulty":{"available":true,"value":151.0}},{"slug":"senior-swe-bench-basic-solve-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-ale-cli-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-ale-cli-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-near-term-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-near-term-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-full-spectrum-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-full-spectrum-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-last-exam-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-last-exam-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-near-term-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-near-term-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-full-spectrum-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-full-spectrum-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-last-exam-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-ale-unlicensed-last-exam-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agents-last-exam-v1-1","signalToNoise":{"available":true,"value":10.045},"difficulty":{"available":true,"value":171.5}},{"slug":"agents-last-exam-v1-1-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"continual-learning-bench-cl-bench-average-gain","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-blind-spectrum-monitoring-reward","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-blind-spectrum-monitoring-gain","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-codebase-adaptation-reward","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-codebase-adaptation-gain","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-cohort-studies-reward","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-cohort-studies-gain","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-database-exploration-reward","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-database-exploration-gain","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-exploitable-poker-reward","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-exploitable-poker-gain","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-sales-prediction-reward","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"continual-learning-bench-cl-bench-sales-prediction-gain","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"mindcube","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"hpct-refusal-cais","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"pct-political-inconsistency-cais","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-overall","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-security","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-bullshit","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-refactoring","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-debugging","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-generation","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-speed-ttft","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"bridgebench-arena-v3-speed-throughput","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"btf2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"btf2-calibration","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"btf2-refinement","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"deep-research-bench","signalToNoise":{"available":true,"value":2.292},"difficulty":{"available":true,"value":157.0}},{"slug":"deep-research-bench-find-number","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deep-research-bench-derive-number","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deep-research-bench-find-dataset","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deep-research-bench-compile-dataset","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deep-research-bench-populate-reference-class","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deep-research-bench-gather-evidence","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deep-research-bench-validate-claim","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deep-research-bench-find-original-source","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"minebench-conservative-rating","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"design-arena-overall-code-categories","signalToNoise":{"available":true,"value":4.608},"difficulty":{"available":true,"value":147.9}},{"slug":"frontierswe-postgres-sqlite-wire-adapter","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-postgres-sqlite-wire-adapter-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-modular-stack-wan21","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-modular-stack-wan21-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-git-to-zig","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-git-to-zig-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-dart-style-haskell","signalToNoise":{"available":true,"value":35.784},"difficulty":{"available":true,"value":162.8}},{"slug":"frontierswe-dart-style-haskell-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-lua-native-compiler","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-lua-native-compiler-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-frogsgame-rl","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-frogsgame-rl-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-pcqm4mv2-autoresearch","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-pcqm4mv2-autoresearch-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-optimizer-design","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-optimizer-design-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-libexpat-to-x86asm","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-libexpat-to-x86asm-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-ffmpeg-swscale-rewrite","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-ffmpeg-swscale-rewrite-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-pyright-type-checking-optimization","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-pyright-type-checking-optimization-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-notebook-compression","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-notebook-compression-best-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-revideo-perf-opt","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-revideo-perf-opt-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-cranelift-codegen-opt","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-cranelift-codegen-opt-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-dependent-type-checker","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-dependent-type-checker-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-granite-mamba2-inference-optimization","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-granite-mamba2-inference-optimization-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-inference-system-optimization","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"frontierswe-inference-system-optimization-best-at-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"worldvqa-accuracy","signalToNoise":{"available":true,"value":3.639},"difficulty":{"available":true,"value":163.6}},{"slug":"worldvqa-not-attempted","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-correct-given-attempted","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-nature-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-geography-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-culture-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-objects-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-transportation-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-entertainment-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-brands-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-sports-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-people-f-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"worldvqa-force-answer","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"diligence-stack-agent","signalToNoise":{"available":true,"value":13.067},"difficulty":{"available":true,"value":149.2}},{"slug":"diligence-stack-agent-completion","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"diligence-stack-agent-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"diligence-stack-agent-source-grounding","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"diligence-stack-agent-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"diligence-stack-agent-artifact-quality","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"diligence-stack-agent-auditability-process","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"diligence-stack-agent-efficiency","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kernelbench-hard-tflops-ratio","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-hard-tflops-ratio-rtx-pro-6000","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-hard-tflops-ratio-h100","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-hard-tflops-ratio-b200","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-mega","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-mega-best-decode-speedup-rtx-pro-6000","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-mega-best-decode-speedup-h100","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-mega-best-decode-speedup-b200","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelbench-cuda","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"diligence-bench","signalToNoise":{"available":true,"value":5.306},"difficulty":{"available":true,"value":170.8}},{"slug":"emergent-collusion-high-illegality-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-writing-style-diversity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cybergym-pass30","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cybergym-e2e-615-patch-only","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-615-s1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-615-s2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-615-s3","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-615-s4","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-920-patch-only","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-920-s1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-920-s2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-920-s3","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-920-s4","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-userspace","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitgym-v8","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitgym-kernel","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitgym-with-mitigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitgym-userspace-with-mitigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitgym-v8-with-mitigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitgym-kernel-with-mitigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"nextjs-ai-agent-evals","signalToNoise":{"available":true,"value":10.294},"difficulty":{"available":true,"value":118.2}},{"slug":"lmarena-text-arena-french-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-non-english-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-polish-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-exclude-ties-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-expert-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-software-it-services-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-industry-writing-and-literature-and-language-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-life-physical-social-science-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-entertainment-sports-media-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-business-management-financial-ops-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-mathematical-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-legal-government-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-occupational-medicine-healthcare-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-instruction-following-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-multiturn-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-creative-writing-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-hard-prompts-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-hard-prompts-english-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-longer-query-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-english-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-non-english-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-chinese-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-french-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-german-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-spanish-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-russian-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-japanese-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-korean-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-polish-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-exclude-ties-no-style-control-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-text-arena-overall-factuality-25-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-english","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-chinese","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-captioning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-creative-writing","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-diagram","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-entity-recognition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-homework","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-humor","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-english-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-chinese-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-captioning-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-creative-writing-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-diagram-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-entity-recognition-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-homework-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-arena-humor-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-vision-ocr-no-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-html","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-react","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-brand-marketing","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-reference-based-design","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-data-analytics","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-consumer-product","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-gaming","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-simulations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-content-creation-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-image-to-webdev-arena","signalToNoise":{"available":true,"value":12.622},"difficulty":{"available":true,"value":157.0}},{"slug":"arena-ai-document-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-search-arena-style-control","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lmarena-search-arena-factuality-25","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"formationeval","signalToNoise":{"available":true,"value":6.857},"difficulty":{"available":true,"value":100.9}},{"slug":"obviousbench","signalToNoise":{"available":true,"value":2.952},"difficulty":{"available":true,"value":118.5}},{"slug":"buseybench-svg","signalToNoise":{"available":true,"value":3.424},"difficulty":{"available":true,"value":157.3}},{"slug":"pointerbench","signalToNoise":{"available":true,"value":4.111},"difficulty":{"available":true,"value":163.4}},{"slug":"pointerbench-sheets","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"pointerbench-text","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"pointerbench-pro","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruby-llm-program-fixer","signalToNoise":{"available":true,"value":1.849},"difficulty":{"available":true,"value":113.4}},{"slug":"ruby-llm-program-fixer-calendar","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruby-llm-program-fixer-parking-garage","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruby-llm-program-fixer-school-library","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruby-llm-program-fixer-vending-machine","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"chessbench","signalToNoise":{"available":true,"value":3.655},"difficulty":{"available":true,"value":160.0}},{"slug":"chessbench-mate-in-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"chessbench-mate-in-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"chessbench-fork","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"chessbench-pin","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"chessbench-hanging-piece","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eval-connections","signalToNoise":{"available":true,"value":3.317},"difficulty":{"available":true,"value":138.2}},{"slug":"eval-connections-guess-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-code-execution","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-stl-creation","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-watertight","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-component-count","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-bounding-box","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-volume","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-chamfer-distance","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cadqueryeval-hausdorff-95p","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"specieval","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-speciesism","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-animal-sentience","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-land-animal-4ns","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-sea-animal-4ns","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"wordlebench","signalToNoise":{"available":true,"value":2.291},"difficulty":{"available":true,"value":138.1}},{"slug":"wordlebench-average-guesses-successful","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"afantasia","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"afantasia-chess","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"afantasia-cube","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"afantasia-spell","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"sm-bench","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-overfit","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-eq-boundaries","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-creative-writing","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-nsfw","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-nsfw-system","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-ambiguous","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-adversarial","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"sm-bench-anti-hallucination","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"ahk-eval","signalToNoise":{"available":true,"value":7.258},"difficulty":{"available":true,"value":144.0}},{"slug":"ahk-eval-hidden-cases","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-parse-success","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-strings","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-regex","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-data","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-numbers","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-datetime","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-algorithms","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-mid","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ahk-eval-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cve-bench-caisi-15","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"ctf-archive-caisi-577","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"ctf-archive-diamond","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"breakpoint-caisi-200","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mmlu-pro-caisi-science-1000","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmmlu-caisi-1400","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"smt-2025-caisi-58","signalToNoise":{"available":true,"value":10.706},"difficulty":{"available":true,"value":122.4}},{"slug":"otis-aime-2025-caisi-30","signalToNoise":{"available":true,"value":16.86},"difficulty":{"available":true,"value":135.4}},{"slug":"pumac-2024-caisi-55","signalToNoise":{"available":true,"value":18.431},"difficulty":{"available":true,"value":134.1}},{"slug":"arc-agi-2-semi-private-caisi-mean","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"caisi-portbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitbench-caisi-inspect-autonudge","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hijackeval-public-attacks-best-of-31","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-cyber-safeguards-no-jailbreak","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-cyber-safeguards-best-public-jailbreak","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"exploitbench-caisi-safeguard-blocking-10","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-biology-safeguards-no-jailbreak","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-biology-safeguards-best-public-jailbreak","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"ccp-narrative-bench-english","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"ccp-narrative-bench-chinese","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-agentdojo-workspace-credential-exfiltration-2025","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-agentdojo-workspace-phishing-emails-2025","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-agentdojo-workspace-malware-execution-2025","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-agentdojo-workspace-credential-exfiltration-non-public-injection-2025","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-agentdojo-workspace-phishing-emails-non-public-injection-2025","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-agentdojo-workspace-malware-execution-non-public-injection-2025","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-harmful-biology-violent-activities-2025-best-public-jailbreak-compliance","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-harmful-biology-violent-activities-2025-best-public-jailbreak-high-detail","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-malicious-hacking-online-scams-2025-best-public-jailbreak-compliance","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"caisi-malicious-hacking-online-scams-2025-best-public-jailbreak-high-detail","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"market-bench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"market-bench-single-stock-scheduled-execution","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"market-bench-single-stock-scheduled-execution-best-of-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"market-bench-pairs-mean-reversion","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"market-bench-pairs-mean-reversion-best-of-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"market-bench-delta-hedging","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"market-bench-delta-hedging-best-of-5","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"app-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"financeqa-assumption-based","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"time-horizon-index-ksp","signalToNoise":{"available":true,"value":19.257},"difficulty":{"available":true,"value":166.3}},{"slug":"time-horizon-index-ksp-progress-efficiency","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"macaron-chatbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"macaron-livingbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vita-bench","signalToNoise":{"available":true,"value":5.35},"difficulty":{"available":true,"value":155.7}},{"slug":"vita-bench-2-rewrite-agentic-memory","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"tau3-bench","signalToNoise":{"available":true,"value":1.486},"difficulty":{"available":true,"value":102.3}},{"slug":"pinchbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"pinchbench-best-score","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"clawgym-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"deepswe-datacurve-2-best-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-atlas-codebase-qna-scale-ai-2-pass-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ui4a-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"swe-bench-multilingual","signalToNoise":{"available":true,"value":6.408},"difficulty":{"available":true,"value":141.8}},{"slug":"mle-bench-partial-30-average-position","signalToNoise":{"available":true,"value":4.649},"difficulty":{"available":true,"value":157.4}},{"slug":"mrcr-v2-8-needle-128k-average","signalToNoise":{"available":true,"value":5.56},"difficulty":{"available":true,"value":144.3}},{"slug":"mrcr-v2-8-needle-1m-pointwise","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"big-sleep-vulnerability-evaluation","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"chrome-production-commit-scanning","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"v8-unique-confirmed-security-issues","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"concept-synth-abd-v1-2","signalToNoise":{"available":true,"value":2.917},"difficulty":{"available":true,"value":110.3}},{"slug":"concept-synth-abd-v1-2-full","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"concept-synth-abd-v1-2-partial","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"concept-synth-abd-v1-2-skeptical","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"concept-synth-induction-v1","signalToNoise":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."},"difficulty":{"available":false,"reasonCode":"no_scores","explanation":"No usable scores."}},{"slug":"concept-synth-induction-v1-fullobs","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"concept-synth-induction-v1-ci","signalToNoise":{"available":true,"value":2.329},"difficulty":{"available":true,"value":136.8}},{"slug":"concept-synth-induction-v1-ec","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"concept-synth-induction-challenge64-round1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"concept-synth-induction-challenge100-round1","signalToNoise":{"available":true,"value":9.747},"difficulty":{"available":true,"value":168.0}},{"slug":"concept-synth-induction-challenge64-holdout","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"concept-synth-induction-challenge100-holdout","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"concept-synth-induction-challenge64-holdout-v2","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"concept-synth-induction-challenge100-holdout-v2","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cred-v0-92","signalToNoise":{"available":true,"value":5.666},"difficulty":{"available":true,"value":144.0}},{"slug":"cred-v0-92-paper-code","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cred-v0-92-prose-table","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cred-v0-92-data-integrity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cred-v0-92-execution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cybergym-e2e","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mathvista","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mathvista-test","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"solo-bench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"taxcalcbench-ty24","signalToNoise":{"available":true,"value":6.329},"difficulty":{"available":true,"value":158.8}},{"slug":"taxcalcbench-ty25","signalToNoise":{"available":true,"value":11.269},"difficulty":{"available":true,"value":174.9}},{"slug":"taxcalcbench-ty25-web-search","signalToNoise":{"available":true,"value":12.257},"difficulty":{"available":true,"value":166.6}},{"slug":"gamedevbench","signalToNoise":{"available":true,"value":5.049},"difficulty":{"available":true,"value":158.2}},{"slug":"edit-bench","signalToNoise":{"available":true,"value":2.168},"difficulty":{"available":true,"value":137.5}},{"slug":"edit-bench-core","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"edit-bench-complete-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"edit-bench-complete-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"edit-bench-core-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"edit-bench-core-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals","signalToNoise":{"available":true,"value":3.267},"difficulty":{"available":true,"value":120.5}},{"slug":"roboflow-vision-evals-object-detection","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-object-detection-map75","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-object-detection-map50-95","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-counting","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-identification","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-ocr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-data-extraction","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-reasoning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontier-bench-0-1","signalToNoise":{"available":true,"value":19.195},"difficulty":{"available":true,"value":170.5}},{"slug":"healthbench-professional-length-adjusted","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"biomysterybench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"biomysterybench-human-difficult","signalToNoise":{"available":true,"value":4.334},"difficulty":{"available":true,"value":166.8}},{"slug":"biomysterybench-human-solvable","signalToNoise":{"available":true,"value":6.653},"difficulty":{"available":true,"value":123.2}},{"slug":"swe-bench-multimodal","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1","signalToNoise":{"available":true,"value":6.937},"difficulty":{"available":true,"value":165.2}},{"slug":"swe-bench-multimodal-vals-v1-bpmn-js","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-carbon","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-eslint","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-fusion-next","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-grommet","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-highlight-js","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-lighthouse","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-openlayers","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-prettier","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-prism","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-multimodal-vals-v1-scratch-gui","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1","signalToNoise":{"available":true,"value":7.84},"difficulty":{"available":true,"value":156.4}},{"slug":"cwe-bench-v1-programmatic-pass4","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-judge-pass1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-judge-pass4","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a01","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a02","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a03","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a04","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a05","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a06","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a07","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a08","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a09","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-a10","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-c-cpp","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-go","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-java","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-js-ts","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-python","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cwe-bench-v1-rust","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"labbench2","signalToNoise":{"available":true,"value":3.179},"difficulty":{"available":true,"value":120.8}},{"slug":"graphwalks-bfs-up-to-128k","signalToNoise":{"available":true,"value":5.867},"difficulty":{"available":true,"value":132.5}},{"slug":"graphwalks-bfs-256k-to-1m","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lvbench","signalToNoise":{"available":true,"value":3.68},"difficulty":{"available":true,"value":141.4}},{"slug":"terminal-bench-science-0-1-verifier-6x","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"google-real-world-vulnerability-discovery","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"wiz-penetration-test-google-internal","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gray-swan-ipi-attack-success-k15","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"lmarena-agent-arena-bash-reliability","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"healthbench-professional","signalToNoise":{"available":true,"value":6.639},"difficulty":{"available":true,"value":151.4}},{"slug":"officeqa","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"oss-fuzz-anthropic","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"firefox-147-exploit-development","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"firefox-147-exploit-development-any-progress","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"latchbio-spatialbench-card","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gdp-pdf-mean-criteria-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"activevision","signalToNoise":{"available":true,"value":4.153},"difficulty":{"available":true,"value":173.0}},{"slug":"activevision-with-tools","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"perceptionbench-visual-relation","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-counting","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-attribute","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-depth-3d","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-localization","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-comparison","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-fine-grained-recognition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-context-integration","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-ocr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"perceptionbench-hallucination","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"pg-llm-50","signalToNoise":{"available":true,"value":22.471},"difficulty":{"available":true,"value":172.6}},{"slug":"ifeval-openllm-harness","signalToNoise":{"available":true,"value":1.261},"difficulty":{"available":true,"value":105.3}},{"slug":"bbh-openllm-harness","signalToNoise":{"available":true,"value":4.008},"difficulty":{"available":true,"value":101.7}},{"slug":"math-lvl-5-openllm-harness","signalToNoise":{"available":true,"value":4.577},"difficulty":{"available":true,"value":130.7}},{"slug":"gpqa-openllm-harness","signalToNoise":{"available":true,"value":13.004},"difficulty":{"available":true,"value":153.5}},{"slug":"musr-openllm-harness","signalToNoise":{"available":true,"value":1.547},"difficulty":{"available":true,"value":151.1}},{"slug":"mmlu-pro-openllm-harness","signalToNoise":{"available":true,"value":10.127},"difficulty":{"available":true,"value":123.6}},{"slug":"arc-challenge-openllm-v1-harness","signalToNoise":{"available":true,"value":4.728},"difficulty":{"available":true,"value":103.3}},{"slug":"hellaswag-openllm-v1-harness","signalToNoise":{"available":true,"value":2.972},"difficulty":{"available":true,"value":70.4}},{"slug":"mmlu-openllm-v1-harness","signalToNoise":{"available":true,"value":9.389},"difficulty":{"available":true,"value":98.9}},{"slug":"truthfulqa-mc2-openllm-v1-harness","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"winogrande-openllm-v1-harness","signalToNoise":{"available":true,"value":4.368},"difficulty":{"available":true,"value":98.5}},{"slug":"gsm8k-openllm-v1-harness","signalToNoise":{"available":true,"value":2.088},"difficulty":{"available":true,"value":102.1}},{"slug":"chessbench-ai-coherence","signalToNoise":{"available":true,"value":1.681},"difficulty":{"available":true,"value":142.3}},{"slug":"chessbench-ai-accuracy","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"chessbench-ai-elo","signalToNoise":{"available":true,"value":2.401},"difficulty":{"available":true,"value":150.0}},{"slug":"bigcodebench-complete","signalToNoise":{"available":true,"value":2.011},"difficulty":{"available":true,"value":118.8}},{"slug":"bigcodebench-instruct","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"bigcodebench-hard-complete","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"bigcodebench-hard-instruct","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmstar-vlmevalkit","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmmu-val-vlmevalkit","signalToNoise":{"available":true,"value":4.492},"difficulty":{"available":true,"value":108.4}},{"slug":"mathvista-vlmevalkit","signalToNoise":{"available":true,"value":2.957},"difficulty":{"available":true,"value":104.2}},{"slug":"ai2d-vlmevalkit","signalToNoise":{"available":true,"value":2.597},"difficulty":{"available":true,"value":80.9}},{"slug":"hallusionbench-vlmevalkit","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmvet-vlmevalkit","signalToNoise":{"available":true,"value":3.129},"difficulty":{"available":true,"value":100.7}},{"slug":"bridge-medical-zero-shot","signalToNoise":{"available":true,"value":3.253},"difficulty":{"available":true,"value":152.1}},{"slug":"bridge-medical-few-shot","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"bridge-medical-cot","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"medic-closed-ended","signalToNoise":{"available":true,"value":1.073},"difficulty":{"available":true,"value":95.1}},{"slug":"medic-summarization","signalToNoise":{"available":true,"value":1.899},"difficulty":{"available":true,"value":5.0}},{"slug":"medic-open-ended-elo","signalToNoise":{"available":true,"value":1.803},"difficulty":{"available":true,"value":118.5}},{"slug":"pubmedqa-openmedical-harness","signalToNoise":{"available":true,"value":1.61},"difficulty":{"available":true,"value":85.7}},{"slug":"medqa-4options-openmedical-harness","signalToNoise":{"available":true,"value":22.916},"difficulty":{"available":true,"value":105.2}},{"slug":"medmcqa-openmedical-harness","signalToNoise":{"available":true,"value":11.79},"difficulty":{"available":true,"value":110.5}},{"slug":"mmlu-anatomy-openmedical-harness","signalToNoise":{"available":true,"value":10.72},"difficulty":{"available":true,"value":102.8}},{"slug":"mmlu-clinical-knowledge-openmedical-harness","signalToNoise":{"available":true,"value":12.935},"difficulty":{"available":true,"value":94.8}},{"slug":"mmlu-college-biology-openmedical-harness","signalToNoise":{"available":true,"value":15.661},"difficulty":{"available":true,"value":94.2}},{"slug":"mmlu-college-medicine-openmedical-harness","signalToNoise":{"available":true,"value":15.489},"difficulty":{"available":true,"value":99.9}},{"slug":"mmlu-medical-genetics-openmedical-harness","signalToNoise":{"available":true,"value":14.576},"difficulty":{"available":true,"value":93.1}},{"slug":"mmlu-professional-medicine-openmedical-harness","signalToNoise":{"available":true,"value":11.828},"difficulty":{"available":true,"value":97.2}},{"slug":"alpacaeval-2-lc-winrate","signalToNoise":{"available":true,"value":4.546},"difficulty":{"available":true,"value":126.4}},{"slug":"alpacaeval-2-winrate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livecodebench-pro","signalToNoise":{"available":true,"value":4.863},"difficulty":{"available":true,"value":141.4}},{"slug":"facts-benchmark-suite","signalToNoise":{"available":true,"value":3.014},"difficulty":{"available":true,"value":148.9}},{"slug":"facts-benchmark-suite-grounding-v3-parametric-v2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"tau2-bench-retail","signalToNoise":{"available":true,"value":3.767},"difficulty":{"available":true,"value":130.0}},{"slug":"global-piqa","signalToNoise":{"available":true,"value":7.028},"difficulty":{"available":true,"value":98.6}},{"slug":"global-piqa-nonparallel-exact-match","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"global-piqa-nonparallel-exact-match-strict","signalToNoise":{"available":true,"value":4.015},"difficulty":{"available":true,"value":132.6}},{"slug":"global-piqa-nonparallel-loglikelihood-accuracy","signalToNoise":{"available":true,"value":5.867},"difficulty":{"available":true,"value":153.9}},{"slug":"global-piqa-parallel-exact-match","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"global-piqa-parallel-exact-match-strict","signalToNoise":{"available":true,"value":4.699},"difficulty":{"available":true,"value":135.7}},{"slug":"global-piqa-parallel-loglikelihood-accuracy","signalToNoise":{"available":true,"value":13.118},"difficulty":{"available":true,"value":166.5}},{"slug":"screenspot-pro","signalToNoise":{"available":true,"value":1.274},"difficulty":{"available":true,"value":142.6}},{"slug":"deepsecbench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-analyst-agent-pass-caret-5","signalToNoise":{"available":true,"value":7.9},"difficulty":{"available":true,"value":164.6}},{"slug":"aa-analyst-agent-pass-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-analyst-agent-pass-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdp-pdf-all-pass-artificial-analysis","signalToNoise":{"available":true,"value":11.559},"difficulty":{"available":true,"value":175.8}},{"slug":"gdp-pdf-mean-criteria-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mlcr-aa","signalToNoise":{"available":true,"value":5.095},"difficulty":{"available":true,"value":168.9}},{"slug":"mlcr-aa-accuracy","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mlcr-aa-completeness","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mlcr-aa-conciseness","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mlcr-aa-2026-10","signalToNoise":{"available":true,"value":5.199},"difficulty":{"available":true,"value":181.3}},{"slug":"mlcr-aa-2026-10-accuracy","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mlcr-aa-2026-10-completeness","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"terminal-bench-4-0-artificial-analysis","signalToNoise":{"available":true,"value":17.75},"difficulty":{"available":true,"value":165.8}},{"slug":"aa-intelligence-index-v4-3","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"scicode-1-0-1-artificial-analysis","signalToNoise":{"available":true,"value":5.45},"difficulty":{"available":true,"value":156.4}},{"slug":"aa-lcr-1-1-artificial-analysis","signalToNoise":{"available":true,"value":5.077},"difficulty":{"available":true,"value":140.4}},{"slug":"automationbench-aa-1-0-6","signalToNoise":{"available":true,"value":9.332},"difficulty":{"available":true,"value":160.5}},{"slug":"arxivmath-08-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"brokenarxiv-08-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-09-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"brokenarxiv-09-2026-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-1-0-6-zapier","signalToNoise":{"available":true,"value":10.89},"difficulty":{"available":true,"value":170.4}},{"slug":"automationbench-1-0-6-sales-zapier","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"automationbench-1-0-6-marketing-zapier","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"automationbench-1-0-6-operations-zapier","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"automationbench-1-0-6-support-zapier","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"automationbench-1-0-6-finance-zapier","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"automationbench-1-0-6-hr-zapier","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"zerobench-main-mean-pass-1-five-samples","signalToNoise":{"available":true,"value":13.33},"difficulty":{"available":true,"value":177.6}},{"slug":"gdpval-aa-v2-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-briefcase-v1-1","signalToNoise":{"available":true,"value":5.553},"difficulty":{"available":true,"value":153.5}},{"slug":"aa-intelligence-index-v4-3-2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"terminal-bench-science-0-1","signalToNoise":{"available":true,"value":38.646},"difficulty":{"available":true,"value":166.4}},{"slug":"vending-bench-arena-round-13","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"ioi-vals-v2","signalToNoise":{"available":true,"value":12.023},"difficulty":{"available":true,"value":159.8}},{"slug":"ioi-vals-v2-2024","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ioi-vals-v2-2025","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ioi-vals-v2-2026","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"proofbench-v1-1","signalToNoise":{"available":true,"value":12.49},"difficulty":{"available":true,"value":159.1}},{"slug":"cyberbench-v1-1-overall","signalToNoise":{"available":true,"value":1.957},"difficulty":{"available":true,"value":138.2}},{"slug":"cyberbench-v1-1-poc","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cyberbench-v1-1-patch","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-index-v2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"code-migration-vals-index-v2-subset","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cursorbench-4-0","signalToNoise":{"available":true,"value":20.72},"difficulty":{"available":true,"value":165.4}},{"slug":"btf3-2386-pooled","signalToNoise":{"available":true,"value":12.461},"difficulty":{"available":true,"value":112.0}},{"slug":"btf3-2386-binary","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"btf3-2386-numeric","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0","signalToNoise":{"available":true,"value":19.2},"difficulty":{"available":true,"value":165.2}},{"slug":"minebench-bradley-terry-rating","signalToNoise":{"available":true,"value":9.556},"difficulty":{"available":true,"value":154.4}},{"slug":"dayjob-finance","signalToNoise":{"available":true,"value":28.31},"difficulty":{"available":true,"value":171.5}},{"slug":"dayjob-healthcare","signalToNoise":{"available":true,"value":26.256},"difficulty":{"available":true,"value":173.5}},{"slug":"hle-diamond","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-bench-pro-v2","signalToNoise":{"available":true,"value":7.889},"difficulty":{"available":true,"value":121.8}},{"slug":"swe-bench-pro-v2-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deepsecbench-revalidated-v1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"deepsecbench-revalidated-v1-233","signalToNoise":{"available":true,"value":8.095},"difficulty":{"available":true,"value":174.4}},{"slug":"biosecbench-surveillance-v1-102","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"spatialbench-v1-verified","signalToNoise":{"available":true,"value":5.129},"difficulty":{"available":true,"value":137.3}},{"slug":"spatialbench-long-v1-verified","signalToNoise":{"available":true,"value":5.876},"difficulty":{"available":true,"value":180.8}},{"slug":"scbench-long-v1-verified","signalToNoise":{"available":true,"value":6.652},"difficulty":{"available":true,"value":175.0}},{"slug":"txbench-ab-v1","signalToNoise":{"available":true,"value":6.76},"difficulty":{"available":true,"value":166.2}},{"slug":"txbench-od-v1","signalToNoise":{"available":true,"value":8.108},"difficulty":{"available":true,"value":168.7}},{"slug":"biosecbench-function-v1","signalToNoise":{"available":true,"value":8.443},"difficulty":{"available":true,"value":169.3}},{"slug":"spatialbench-v2","signalToNoise":{"available":true,"value":9.893},"difficulty":{"available":true,"value":150.9}},{"slug":"spatialbench-long-v2","signalToNoise":{"available":true,"value":4.041},"difficulty":{"available":true,"value":186.2}},{"slug":"scbench-v2","signalToNoise":{"available":true,"value":5.287},"difficulty":{"available":true,"value":157.2}},{"slug":"scbench-long-v2","signalToNoise":{"available":true,"value":4.758},"difficulty":{"available":true,"value":187.4}},{"slug":"epibench-v2","signalToNoise":{"available":true,"value":7.572},"difficulty":{"available":true,"value":182.4}},{"slug":"variantbench-v2","signalToNoise":{"available":true,"value":9.584},"difficulty":{"available":true,"value":167.7}},{"slug":"txbench-ab-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"txbench-pp-v2","signalToNoise":{"available":true,"value":2.73},"difficulty":{"available":true,"value":159.5}},{"slug":"txbench-od-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"biosecbench-surveillance-v2","signalToNoise":{"available":true,"value":8.215},"difficulty":{"available":true,"value":164.6}},{"slug":"biosecbench-refusal-v2","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"biosecbench-function-v2","signalToNoise":{"available":true,"value":12.114},"difficulty":{"available":true,"value":165.7}},{"slug":"metagenomicsbench-v0","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"metagenomicsbench-v1","signalToNoise":{"available":true,"value":6.519},"difficulty":{"available":true,"value":168.5}},{"slug":"osworld-2-0808-full-500-binary","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"osworld-2-0808-full-500-partial","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"osworld-2-0808-offline-500-binary","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"osworld-2-0808-offline-500-partial","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"osworld-2-1-full-500-binary","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-1-full-500-partial","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-1-offline-500-binary","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"osworld-2-1-offline-500-partial","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"reactbench-39","signalToNoise":{"available":true,"value":13.364},"difficulty":{"available":true,"value":168.2}},{"slug":"reactbench-39-write","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"reactbench-39-fix","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-v2","signalToNoise":{"available":true,"value":30.185},"difficulty":{"available":true,"value":165.7}},{"slug":"frontierswe-v2-best","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierswe-v2-worst","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"apex-agents-1-1","signalToNoise":{"available":true,"value":7.757},"difficulty":{"available":true,"value":153.2}},{"slug":"apex-agents-1-1-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"apex-accounting","signalToNoise":{"available":true,"value":6.249},"difficulty":{"available":true,"value":162.4}},{"slug":"apex-accounting-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"furniture-assembly","signalToNoise":{"available":true,"value":10.918},"difficulty":{"available":true,"value":164.4}},{"slug":"mirrorcode-ml-private-2l","signalToNoise":{"available":true,"value":9.677},"difficulty":{"available":true,"value":164.3}},{"slug":"frontiermath-erdos","signalToNoise":{"available":true,"value":34.917},"difficulty":{"available":true,"value":190.0}},{"slug":"oeis-open-50usd","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"oeis-open-lite-200usd","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"simpleqa-verified-epoch-anti-abstention","signalToNoise":{"available":true,"value":3.064},"difficulty":{"available":true,"value":161.0}},{"slug":"ebr-bench-v3","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"ebr-bench-v4-card-ban","signalToNoise":{"available":true,"value":6.219},"difficulty":{"available":true,"value":162.8}},{"slug":"sec-bench-pro-caisi-183","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitbench-caisi-best-of-3","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-userspace-caisi-498","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"oss-fuzz-caisi-297","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"tlo-100m-steps","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"tlo-100m-completion","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitbench-caisi-ace","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitbench-caisi-july-preliminary","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"biomysterybench-vals","signalToNoise":{"available":true,"value":11.509},"difficulty":{"available":true,"value":151.7}},{"slug":"biomysterybench-vals-human-solvable","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"biomysterybench-vals-human-difficult","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cua-bench","signalToNoise":{"available":true,"value":11.681},"difficulty":{"available":true,"value":189.1}},{"slug":"cua-bench-minecraft-nether","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cua-bench-held-out-sandbox","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cua-bench-superhot","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cua-bench-held-out-reflex","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cua-bench-efootball","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cua-bench-held-out-sports","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mysterymechanism","signalToNoise":{"available":true,"value":10.411},"difficulty":{"available":true,"value":170.6}},{"slug":"srebench","signalToNoise":{"available":true,"value":32.312},"difficulty":{"available":true,"value":169.2}},{"slug":"srebench-capability","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-rsi-index-v1-1","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vals-rsi-index-v1-1-compression","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vals-rsi-index-v1-1-lm-training","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vals-rsi-index-v1-1-parameter-golf","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vals-rsi-index-v1-1-harness-engineering","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vals-rsi-index-v1-1-post-training","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vcb-1-100","signalToNoise":{"available":true,"value":12.612},"difficulty":{"available":true,"value":176.5}},{"slug":"tax-agent-bench","signalToNoise":{"available":true,"value":9.233},"difficulty":{"available":true,"value":155.2}},{"slug":"tax-agent-bench-all-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-fact-pattern-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-rule-source-lookup","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-current-temporal-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-numbers-calculations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-forms-filings","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-controversy-precedence-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-all-pass-fact-pattern-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-all-pass-rule-source-lookup","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-all-pass-current-temporal-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-all-pass-numbers-calculations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-all-pass-forms-filings","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-all-pass-controversy-precedence-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0-software","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0-science","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0-ml","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0-operations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0-hardware","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0-security","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-4-0-media","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-science-vals-retry","signalToNoise":{"available":true,"value":37.643},"difficulty":{"available":true,"value":168.6}},{"slug":"terminal-bench-science-vals-retry-life-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-science-vals-retry-physical-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-science-vals-retry-earth-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-science-vals-retry-mathematical-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"terminal-bench-science-vals-retry-engineering-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09","signalToNoise":{"available":true,"value":2.717},"difficulty":{"available":true,"value":125.7}},{"slug":"roboflow-vision-evals-2026-09-detection-map50","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-detection-map75","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-detection-map50-95","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-counting-judge","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-counting-strict","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"roboflow-vision-evals-2026-09-identification-judge","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-identification-strict","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"roboflow-vision-evals-2026-09-ocr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-extraction-judge","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-extraction-strict","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"roboflow-vision-evals-2026-09-reasoning-judge","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-09-reasoning-strict","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"apex-shortlist-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aime-2026-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arxivlean-03-2026-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arxivmath-01-2026-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arxivmath-02-2026-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arxivmath-03-2026-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arxivmath-12-2025-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hmmt-feb-2026-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kangaroo-2025-5-6-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"smt-2025-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"project-euler-matharena-estimated","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vals-rsi-index-v1-1-four-tasks","signalToNoise":{"available":true,"value":14.508},"difficulty":{"available":true,"value":175.7}},{"slug":"vals-rsi-index-v1-1-four-tasks-compression","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-rsi-index-v1-1-four-tasks-lm-training","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-rsi-index-v1-1-four-tasks-harness-engineering","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-rsi-index-v1-1-four-tasks-post-training","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vals-index-v2-1","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vals-index-v2-2","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"code-migration-v1-1","signalToNoise":{"available":true,"value":14.518},"difficulty":{"available":true,"value":160.7}},{"slug":"code-migration-v1-1-cli","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"code-migration-v1-1-cobol","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"code-migration-v1-1-code-quality","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"code-migration-v1-1-vals-index-subset","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1","signalToNoise":{"available":true,"value":10.469},"difficulty":{"available":true,"value":149.9}},{"slug":"emb-v1-1-template","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-scratch","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-lbo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-dcf","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-m-a","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-operating","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-comps","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-dataroom","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"emb-v1-1-misc","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1","signalToNoise":{"available":true,"value":6.65},"difficulty":{"available":true,"value":149.1}},{"slug":"finance-agent-v2-1-all-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-general-qualitative-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-general-quantitative-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-market-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-comparables","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-precedents","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-adjustments","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-earnings-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-disclosure-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v2-1-financial-modeling","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"claimbench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"claimbench-direct-support","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"claimbench-subtle-contradiction","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"claimbench-numeric-discrepancy","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"claimbench-missing-evidence","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hlab-v1-1","signalToNoise":{"available":true,"value":3.332},"difficulty":{"available":true,"value":201.4}},{"slug":"hlab-v1-1-criteria-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-antitrust-competition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-arbitration-international-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-banking-finance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-bankruptcy-restructuring","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-capital-markets","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-corporate-governance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-corporate-ma","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-data-privacy-cybersecurity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-emerging-companies-venture-capital","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-employment-labor","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-energy-natural-resources","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-environmental-esg","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-funds-asset-management","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-healthcare-life-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-immigration","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-insurance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-intellectual-property","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-international-trade-sanctions","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-litigation-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-real-estate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-structured-finance-securitization","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-tax","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-trusts-estates-private-client","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-white-collar-defense-investigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-antitrust-competition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-arbitration-international-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-banking-finance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-bankruptcy-restructuring","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-capital-markets","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-corporate-governance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-corporate-ma","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-data-privacy-cybersecurity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-emerging-companies-venture-capital","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-employment-labor","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-energy-natural-resources","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-environmental-esg","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-funds-asset-management","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-healthcare-life-sciences","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-immigration","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-insurance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-intellectual-property","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-international-trade-sanctions","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-litigation-dispute-resolution","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-real-estate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-structured-finance-securitization","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-tax","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-trusts-estates-private-client","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-v1-1-criteria-pass-rate-white-collar-defense-investigations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"claimbench-non-supporting-citation","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-net-improvement-steering-burden","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"lmarena-agent-arena-steering-burden","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gdp-xlsx","signalToNoise":{"available":true,"value":9.9},"difficulty":{"available":true,"value":179.0}},{"slug":"librarydesignbench","signalToNoise":{"available":true,"value":7.848},"difficulty":{"available":true,"value":171.0}},{"slug":"librarydesignbench-pass-rate","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"librarydesignbench-simplicity","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"extended-nyt-connections-940-quadratic-v1","signalToNoise":{"available":true,"value":5.669},"difficulty":{"available":true,"value":150.0}},{"slug":"extended-nyt-connections-newest-100-quadratic-v1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"llm-sycophancy-balanced-198","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-balanced-198-conditional","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-balanced-198-decisive-coverage","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-balanced-198-stripped","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-balanced-198-insufficient","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-balanced-198-total-inconsistency","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-balanced-198-conditional-total-inconsistency","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"llm-sycophancy-balanced-198-contrarian","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"afantasia-valid-retry","signalToNoise":{"available":true,"value":1.421},"difficulty":{"available":true,"value":150.9}},{"slug":"afantasia-valid-retry-chess","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"afantasia-valid-retry-cube","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"afantasia-valid-retry-spell","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"specieval-valid-retry","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-valid-retry-speciesism","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-valid-retry-animal-sentience","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-valid-retry-land-animal-4ns","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-valid-retry-sea-animal-4ns","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-english-pooled","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-english-pooled-speciesism","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-english-pooled-animal-sentience","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-english-pooled-land-animal-4ns","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"specieval-english-pooled-sea-animal-4ns","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"cadqueryeval-corrected-geometry","signalToNoise":{"available":true,"value":4.456},"difficulty":{"available":true,"value":150.2}},{"slug":"cadqueryeval-corrected-geometry-code-executed","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval-corrected-geometry-stl-created","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval-corrected-geometry-watertight","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval-corrected-geometry-single-component","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval-corrected-geometry-bbox","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval-corrected-geometry-volume","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval-corrected-geometry-chamfer","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cadqueryeval-corrected-geometry-hausdorff","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kernelbench-hard-normalized-rtx-pro-6000","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kernelbench-hard-normalized-h100","signalToNoise":{"available":true,"value":2.53},"difficulty":{"available":true,"value":151.7}},{"slug":"kernelbench-hard-normalized-b200","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kernelbench-cuda-normalized-rtx-pro-6000","signalToNoise":{"available":true,"value":8.56},"difficulty":{"available":true,"value":165.1}},{"slug":"kernelbench-cuda-normalized-h100","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kernelbench-cuda-normalized-b200","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kernelbench-mega-normalized-rtx-pro-6000","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"kernelbench-mega-normalized-h100","signalToNoise":{"available":true,"value":6.797},"difficulty":{"available":true,"value":167.9}},{"slug":"kernelbench-mega-normalized-b200","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eval-connections-one-shot-traps","signalToNoise":{"available":true,"value":3.684},"difficulty":{"available":true,"value":144.2}},{"slug":"eval-connections-one-shot-traps-solves","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eval-connections-one-shot-traps-groups","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eval-connections-one-shot-traps-bonus","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"zerobench-main-questions-pass-1-with-tools","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"zerobench-subquestions-pass-1-with-tools","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"posttrainbench-v1-2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"snorkel-agentic-coding-2","signalToNoise":{"available":true,"value":12.091},"difficulty":{"available":true,"value":170.0}},{"slug":"snorkel-agentic-coding-2-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-swe-bench-cli","signalToNoise":{"available":true,"value":31.732},"difficulty":{"available":true,"value":174.9}},{"slug":"snorkel-swe-bench-cli-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-finance-2","signalToNoise":{"available":true,"value":4.761},"difficulty":{"available":true,"value":193.7}},{"slug":"snorkel-finance-2-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-underwrite-2","signalToNoise":{"available":true,"value":4.474},"difficulty":{"available":true,"value":196.2}},{"slug":"snorkel-underwrite-2-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-manufacturing","signalToNoise":{"available":true,"value":5.452},"difficulty":{"available":true,"value":195.2}},{"slug":"snorkel-manufacturing-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-revops","signalToNoise":{"available":true,"value":7.774},"difficulty":{"available":true,"value":195.9}},{"slug":"snorkel-revops-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-legal","signalToNoise":{"available":true,"value":2.768},"difficulty":{"available":true,"value":185.4}},{"slug":"snorkel-legal-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-workplace-agents","signalToNoise":{"available":true,"value":9.422},"difficulty":{"available":true,"value":198.2}},{"slug":"snorkel-workplace-agents-pass-5","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-digital-work-index","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"snorkel-digital-work-index-coding","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"humanitys-sixth-sense","signalToNoise":{"available":true,"value":6.367},"difficulty":{"available":true,"value":172.4}},{"slug":"lmarena-text-to-image-arena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-to-image-arena-commercial-design","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-to-image-arena-3d-modeling","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-to-image-arena-cartoon","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-to-image-arena-photorealistic","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-to-image-arena-art","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-to-image-arena-portraits","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-text-to-image-arena-text-rendering","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-image-edit-arena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-multi-image-edit-arena","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"tax-agent-bench-citation-adjusted-all-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-citation-adjusted-all-pass-fact-pattern-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-citation-adjusted-all-pass-rule-source-lookup","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-citation-adjusted-all-pass-current-temporal-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-citation-adjusted-all-pass-numbers-calculations","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-citation-adjusted-all-pass-forms-filings","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tax-agent-bench-citation-adjusted-all-pass-controversy-precedence-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sudo-l7-surge-ai","signalToNoise":{"available":true,"value":14.143},"difficulty":{"available":true,"value":173.6}},{"slug":"mbabench-2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-easy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-medium","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-medium-hard","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-hard","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-numerical-accuracy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-easy-numerical-accuracy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-medium-numerical-accuracy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-medium-hard-numerical-accuracy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mbabench-2-hard-numerical-accuracy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"txbench-ab-v1-100","signalToNoise":{"available":true,"value":2.426},"difficulty":{"available":true,"value":172.4}},{"slug":"txbench-od-v1-113","signalToNoise":{"available":true,"value":1.533},"difficulty":{"available":true,"value":180.8}},{"slug":"roboflow-vision-evals-2026-10","signalToNoise":{"available":true,"value":2.34},"difficulty":{"available":true,"value":137.5}},{"slug":"roboflow-vision-evals-2026-10-without-ocr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-10-ocr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-10-ocr-single-value","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-10-ocr-transcription","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-10-ocr-structured","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"roboflow-vision-evals-2026-10-ocr-localization","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"distressbench","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-urgent-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-urgent-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-health-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-health-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-relationships-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-relationships-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"epoch-automation-reports","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"epoch-automation-reports-graphic-design","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"epoch-automation-reports-data-insights","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"epoch-automation-reports-data-explorers","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"epoch-automation-reports-data-center-research","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"epoch-automation-reports-research-design","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"arena-alignment-index","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"arena-alignment-unauthorized-action","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"arena-alignment-false-attribution","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"arena-alignment-deceptive-completion","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"reactbench-39-write-28","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"reactbench-39-fix-11","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-3d-v1","signalToNoise":{"available":true,"value":13.062},"difficulty":{"available":true,"value":161.6}},{"slug":"frontiercode-1-1-main-aa","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-marathon-v1-1","signalToNoise":{"available":true,"value":7.838},"difficulty":{"available":true,"value":167.1}},{"slug":"safe-teen-106","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-urgent-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-urgent-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-health-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-health-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-relationships-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-relationships-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-worst-branch","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-worst-branch-urgent-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-worst-branch-urgent-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-worst-branch-health-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-worst-branch-health-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-worst-branch-relationships-no-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"safe-teen-106-worst-branch-relationships-u18-prompt","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"android-bench-2","signalToNoise":{"available":true,"value":20.875},"difficulty":{"available":true,"value":172.4}},{"slug":"android-bench-2-completion","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"astabench-2026","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"astabench-2026-paperfinding","signalToNoise":{"available":true,"value":6.956},"difficulty":{"available":true,"value":173.2}},{"slug":"astabench-2026-litqa-search","signalToNoise":{"available":true,"value":2.195},"difficulty":{"available":true,"value":120.4}},{"slug":"astabench-2026-litqa-fulltext","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"astabench-2026-scholarqa-cs2","signalToNoise":{"available":true,"value":1.774},"difficulty":{"available":true,"value":120.3}},{"slug":"astabench-2026-arxivdigestables","signalToNoise":{"available":true,"value":3.002},"difficulty":{"available":true,"value":188.6}},{"slug":"astabench-2026-super-expert","signalToNoise":{"available":true,"value":3.778},"difficulty":{"available":true,"value":167.9}},{"slug":"astabench-2026-core-hard","signalToNoise":{"available":true,"value":13.159},"difficulty":{"available":true,"value":146.7}},{"slug":"astabench-2026-ds1000","signalToNoise":{"available":true,"value":5.127},"difficulty":{"available":true,"value":124.5}},{"slug":"astabench-2026-discoverybench","signalToNoise":{"available":true,"value":3.16},"difficulty":{"available":true,"value":181.4}},{"slug":"astabench-2026-e2e-hard","signalToNoise":{"available":true,"value":3.455},"difficulty":{"available":true,"value":161.2}},{"slug":"astabench-2026-e2e","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"harbor-index-1-4","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"pg-llm-50-temporal-59","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"oncologybench-strict-25-oct9","signalToNoise":{"available":true,"value":13.482},"difficulty":{"available":true,"value":173.2}},{"slug":"zerobench-subquestions-inspect-2025-05-30","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"zerobench-subquestions-dots-vlm1-report","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"zerobench-main-dots-vlm1-report","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"healthbench-consensus-updated-implementation","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"healthbench-consensus-length-adjusted","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"widesearch-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"deepresearchbench-ii","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"skillsbench-text-79-agentcompass","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"workspace-bench-english-full","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"workspace-bench-english-lite","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mle-bench-lite-humanrank","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gdpval-agentcompass-glm-5-3-elo","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-1-0-6-reported","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tau3-banking-knowledge-alltools-97","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livecodebench-pro-2025-q2-pass-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-interact-75","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"tau2-bench-airline-50","signalToNoise":{"available":true,"value":4.92},"difficulty":{"available":true,"value":137.5}},{"slug":"codeforces-google-deep-think-2026-02","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"ipho-2025-theory","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"icho-2025-theory","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cmt-benchmark-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"imo-2025-reported","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"taxcalcbench-ty25-without-ca-008","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"programbench-almost-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-1-full-reported-partial","signalToNoise":{"available":true,"value":19.772},"difficulty":{"available":true,"value":153.9}},{"slug":"osworld-2-1-full-reported-binary","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-1-offline-reported-partial","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"osworld-2-reported-partial","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"osworld-2-reported-binary","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-aa-reported-strict","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"benchcad-reported-iou","signalToNoise":{"available":true,"value":3.355},"difficulty":{"available":true,"value":157.8}},{"slug":"lifescibench-gold-v1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lifescibench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"genebench-pro-v13","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sec-bench-pro-reported","signalToNoise":{"available":true,"value":19.669},"difficulty":{"available":true,"value":157.8}},{"slug":"exploitbench-reported-progress","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"exploitbench-june-august-2026","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"exploitgym-reported-intended-count","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"exploitgym-reported-all-captures","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"nl2repo-reported","signalToNoise":{"available":true,"value":13.381},"difficulty":{"available":true,"value":156.9}},{"slug":"swe-bench-pro-qwen-corrected","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"claw-eval-mm-pass-3","signalToNoise":{"available":true,"value":11.819},"difficulty":{"available":true,"value":152.2}},{"slug":"claw-eval-mm-average-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vision2web-reported","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mathvision-qwen-corrected","signalToNoise":{"available":true,"value":3.954},"difficulty":{"available":true,"value":120.4}},{"slug":"hle-verified-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gdp-pdf-all-pass-reported","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lvbench-agentic-reported","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"chartqapro-reported","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"graphwalks-parents-256k","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"graphwalks-parents-1m","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"graphwalks-bfs-256k-1m-card-aggregate","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"posttrainbench-lite-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mrcr-v2-8-needle-256k-512k","signalToNoise":{"available":true,"value":8.552},"difficulty":{"available":true,"value":147.1}},{"slug":"mrcr-v2-8-needle-512k-1m","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"eebench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"coworkbench-qwen-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"recreationbench-qwen-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mimo-code-bench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mimo-cyber-bench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mimo-visualcoding-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"programbench-reported-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"dsbench-fullstack-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"dsbench-hard-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"finstepbench-livesearch-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"finstepbench-corporatevaluation-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"finstepbench-financedr-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"stepcodebench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"draco-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"presentbench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"frontierfinance-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"codeforces-deepseek-v4-1-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"sre-bench-openai-reported-pass-1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"terminal-bench-3-reported","signalToNoise":{"available":true,"value":20.834},"difficulty":{"available":true,"value":167.6}},{"slug":"ifbench-reported","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"androidworld-reported","signalToNoise":{"available":true,"value":4.092},"difficulty":{"available":true,"value":134.2}},{"slug":"realworldqa-reported","signalToNoise":{"available":true,"value":2.316},"difficulty":{"available":true,"value":85.1}},{"slug":"jobbench-reported-mean-reward","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arxivmath-march-april-2026-average-4","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-public-anthropic-1235","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"posttrainbench-glm-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"exploitbench-glm-coverage-union-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"swe-marathon-release-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mrcr-v2-8-needle-4k-8k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mrcr-v2-8-needle-8k-16k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mrcr-v2-8-needle-16k-32k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mrcr-v2-8-needle-32k-64k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mrcr-v2-8-needle-64k-128k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mrcr-v2-8-needle-128k-256k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mrcr-v2-8-needle-context-arena-128k-auc","signalToNoise":{"available":true,"value":3.662},"difficulty":{"available":true,"value":151.6}},{"slug":"mrcr-v2-8-needle-context-arena-128k-pointwise","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mrcr-v2-8-needle-context-arena-1m-pointwise","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"finance-agent-v1-1-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"genebench-release-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"frontier-science-research-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"agents-last-exam-reported-pass-rate","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"agents-last-exam-reported-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"jobbench-release-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-v6-reported","signalToNoise":{"available":true,"value":5.231},"difficulty":{"available":true,"value":131.8}},{"slug":"exploitgym-release-reported-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"bixbench-release-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openscore-string-quartets-omr-1-ned","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"harvey-lab-aa-release-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"browsecomp-sonnet-5-multiagent-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-design-tasks-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-data-science-tasks-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-database-migration-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"medchembench-openai-internal-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-expert-swe-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-investment-banking-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-capture-the-flag-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-management-consulting-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"big-finance-bench-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-research-debugging-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kernelgen-1p-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"nanogpt-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"rsi-index-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"automationbench-release-reported-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-internal-computer-use-safety-reported","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"openai-internal-circumvention-reported","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"exploitgym-openai-honeypot-reported","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"impossible-exploitgym-openai-reported","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"openai-internal-hallucination-reported","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"decision-bench-1-1","signalToNoise":{"available":true,"value":29.277},"difficulty":{"available":true,"value":167.2}},{"slug":"decision-bench-1-1-solved","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"build-a-chip-bench","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"build-a-chip-bench-functional-pass","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"build-a-chip-bench-gds-pass","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"real-swe","signalToNoise":{"available":true,"value":5.375},"difficulty":{"available":true,"value":174.1}},{"slug":"real-swe-1-1","signalToNoise":{"available":true,"value":9.455},"difficulty":{"available":true,"value":168.6}},{"slug":"design-creativity-bench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"design-creativity-bench-originality","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"design-creativity-bench-appropriateness","signalToNoise":{"available":true,"value":23.172},"difficulty":{"available":true,"value":123.4}},{"slug":"design-creativity-bench-creative-range","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"autoformbench","signalToNoise":{"available":true,"value":8.65},"difficulty":{"available":true,"value":161.2}},{"slug":"sciuniverse-level-1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"sciuniverse-level-1-chemistry","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"sciuniverse-level-1-biology","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"sciuniverse-level-1-materials","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"sciuniverse-level-1-cross-domain","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"robo-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"robo-bench-arm","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"robo-bench-humanoid","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"robo-bench-balance-scale","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"robo-bench-lego-manual","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"robo-bench-ninja-course","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"physicianbench","signalToNoise":{"available":true,"value":9.527},"difficulty":{"available":true,"value":164.7}},{"slug":"teacharena","signalToNoise":{"available":true,"value":4.833},"difficulty":{"available":true,"value":115.1}},{"slug":"teacharena-pedagogical-judgment","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"teacharena-situated-tutoring","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"teacharena-teaching-workflows","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mazebench-ascii","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mazebench-ascii-python","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mazebench-ascii-rooms","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mazebench-ascii-python-rooms","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"advancedif","signalToNoise":{"available":true,"value":3.546},"difficulty":{"available":true,"value":132.7}},{"slug":"advancedif-rubric-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"browsecomp-mercor-130","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"charxiv-validation-combined","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cvdp-517","signalToNoise":{"available":true,"value":7.857},"difficulty":{"available":true,"value":156.4}},{"slug":"deepresearchbench-ii-mercor-130","signalToNoise":{"available":true,"value":1.427},"difficulty":{"available":true,"value":170.1}},{"slug":"gdpval-mercor-213-rubric","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"medxpertqa-mm-200","signalToNoise":{"available":true,"value":4.745},"difficulty":{"available":true,"value":147.5}},{"slug":"programbench-mercor-100","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"redlinebench","signalToNoise":{"available":true,"value":2.716},"difficulty":{"available":true,"value":170.2}},{"slug":"scicode-main-problems","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"superchem","signalToNoise":{"available":true,"value":4.474},"difficulty":{"available":true,"value":154.1}},{"slug":"frontierswe-v2-mercor","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-cyber-index-v1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cwe-bench-aa-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"deepsecbench-aa-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cybergym-e2e-aa-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"aa-coding-agent-index-v1-5","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"aa-coding-agent-index-v2-0","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lmarena-agent-arena-code-net-improvement-steering-burden","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"lmarena-agent-arena-code-confirmed-success","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-code-praise-vs-complaint","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-code-steering-burden","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-code-bash-reliability","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-code-tool-hallucination","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-chat-net-improvement-steering-burden","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"lmarena-agent-arena-chat-confirmed-success","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-chat-praise-vs-complaint","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-chat-steering-burden","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-chat-bash-reliability","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-chat-tool-hallucination","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-work-net-improvement-steering-burden","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"lmarena-agent-arena-work-confirmed-success","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-work-praise-vs-complaint","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-work-steering-burden","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-work-bash-reliability","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lmarena-agent-arena-work-tool-hallucination","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"webdev-arena-fullstack","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"webdev-arena-frontend","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"dtbench","signalToNoise":{"available":true,"value":10.521},"difficulty":{"available":true,"value":136.3}},{"slug":"lmca","signalToNoise":{"available":true,"value":8.392},"difficulty":{"available":true,"value":158.2}},{"slug":"okapi-multilingual-mmlu","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mmlu-cot-32-uncertainty-routed","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mmlu-cot-8-uncertainty-routed","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gsm8k-maj1-32","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mmmu-maj1-32","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-coding-agent-index-reported-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"programbench-anthropic-166-golden-5-episode","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"biomysterybench-human-solvable-original-set","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"biomysterybench-human-difficult-original-set","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"apex-agents-reported-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arc-agi-1-semi-private","signalToNoise":{"available":true,"value":13.561},"difficulty":{"available":true,"value":147.6}},{"slug":"facts-benchmark-suite-without-search","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"facts-benchmark-suite-without-parametric","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"otis-mock-aime-2024-2025-epoch-model-graded","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gpqa-diamond-average-8-epoch","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aidanbench-62-prompts","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"swe-marathon-v1-1-h20-kimi-k3","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"posttrainbench-pre-v1-1-h100","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"posttrainbench-pre-v1-1-h20-kimi-k3","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"programbench-average-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cybench-anthropic-35-repeated-trials","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"osworld-2-august-2026-anthropic-fixes-500-binary","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"osworld-2-august-2026-anthropic-fixes-500-partial","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"exploitbench-anthropic-capability-union-3","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ctf-archive-diamond-irt-imputed","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2024-10-01-2025-02-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2025-01-01-2025-05-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"swe-bench-verified-openai-477","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"videomme-audio-visual-subtitles","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hle-verified-1811","signalToNoise":{"available":true,"value":3.506},"difficulty":{"available":true,"value":161.2}},{"slug":"osworld-2-pre-august-8-500-step-max-3-partial","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cmt-benchmark-pass-8-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"halluhard-all-domains-average-halluhard-2026-06-15","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aime-2025-matharena","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"chessbench-output-repair","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"chessbench-mate-in-1-output-repair","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"chessbench-mate-in-2-output-repair","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"chessbench-fork-output-repair","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"chessbench-pin-output-repair","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"chessbench-hanging-piece-output-repair","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-april-2026-all-1000","signalToNoise":{"available":true,"value":7.522},"difficulty":{"available":true,"value":143.2}},{"slug":"mcp-atlas-april-2026-public-500","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-april-2026-20-turns","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-gemini-3-1-pro-public","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-kimi-k3-public-500","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-opus-5-protocol-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-step-5-preview-protocol-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-beam-1-0-protocol-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"frontiercs-2","signalToNoise":{"available":true,"value":12.883},"difficulty":{"available":true,"value":170.6}},{"slug":"frontiercs-2-mean-score","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"frontiercs-algorithmic-1-1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"frontiercs-algorithmic-1-0-agent","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mentalhealthbench","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"decepeval-tool-use-induced","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"decepeval-tool-use-neutral","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"decepeval-coding-induced","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"decepeval-coding-neutral","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"decepeval-long-horizon-induced","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"decepeval-long-horizon-neutral","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"rt-safe","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"rt-safe-safe-success-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"rt-safe-success-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"rt-safe-hard-realtime-collisions","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"rt-safe-hard-static-collisions","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"spireclimb","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"spireclimb-sts1-a20-heart","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"spireclimb-sts2-a10","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"hackerrank-bench","signalToNoise":{"available":true,"value":14.585},"difficulty":{"available":true,"value":168.2}},{"slug":"eurekabench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"eurekabench-predictive-accuracy","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"eurekabench-scientific-insights","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"hyperbrowsecomp","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"contractbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"contractbench-completion-rate","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"active-probing-benchmark","signalToNoise":{"available":true,"value":6.122},"difficulty":{"available":true,"value":166.1}},{"slug":"active-probing-benchmark-biology","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"active-probing-benchmark-human-disease","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"active-probing-benchmark-chemistry","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"active-probing-benchmark-physics","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"active-probing-benchmark-mathematics","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"active-probing-benchmark-computer-science","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"active-probing-benchmark-economics","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"active-probing-benchmark-philosophy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench","signalToNoise":{"available":true,"value":11.615},"difficulty":{"available":true,"value":155.6}},{"slug":"aurora-multilingual-terminal-bench-ar","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-cs","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-de","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-es","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-hi","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-ja","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-ko","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-sr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-tr","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-terminal-bench-zh","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aurora-multilingual-tau3-bench-de","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aurora-multilingual-tau3-bench-en","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aurora-multilingual-tau3-bench-fa","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aurora-multilingual-tau3-bench-ko","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aurora-multilingual-tau3-bench-tr","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cyberarena","signalToNoise":{"available":true,"value":4.612},"difficulty":{"available":true,"value":144.0}},{"slug":"worldauditbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"worldauditbench-static-physics","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"worldauditbench-interactive-physics","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"worldauditbench-spatial-consistency","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"worldauditbench-temporal-consistency","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"worldauditbench-semantic-consistency","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"learn2play-bench","signalToNoise":{"available":true,"value":5.438},"difficulty":{"available":true,"value":150.3}},{"slug":"learn2play-bench-mean","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hermes-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"hermes-bench","signalToNoise":{"available":true,"value":5.827},"difficulty":{"available":true,"value":139.6}},{"slug":"aa-robotics","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"culturalriddles","signalToNoise":{"available":true,"value":3.318},"difficulty":{"available":true,"value":146.1}},{"slug":"culturalriddles-exact-match","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"swe-sweep","signalToNoise":{"available":true,"value":15.888},"difficulty":{"available":true,"value":196.9}},{"slug":"sweeper-bench","signalToNoise":{"available":true,"value":3.209},"difficulty":{"available":true,"value":162.0}},{"slug":"opends-world","signalToNoise":{"available":true,"value":5.947},"difficulty":{"available":true,"value":177.5}},{"slug":"opends-world-low-difficulty","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"opends-world-medium-difficulty","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"opends-world-high-difficulty","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agentvqa","signalToNoise":{"available":true,"value":3.766},"difficulty":{"available":true,"value":177.8}},{"slug":"agentvqa-web-agents","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agentvqa-egocentric-videos","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agentvqa-games","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agentvqa-spatial-planning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"agentvqa-robotics","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blenderworld","signalToNoise":{"available":true,"value":4.48},"difficulty":{"available":true,"value":154.7}},{"slug":"blenderworld-text","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"blenderworld-multimodal","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"into-the-unknown","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"into-the-unknown-integrity-nodes","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"into-the-unknown-direction-nodes","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"lscbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lscbench-function","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lscbench-file","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lscbench-repo","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"live-bench-2","signalToNoise":{"available":true,"value":9.718},"difficulty":{"available":true,"value":150.8}},{"slug":"livebench-2-long-agentic","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-2-reasoning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-2-coding","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-2-browser","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-2-data-analysis","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-2-research","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livebench-2-design","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ple","signalToNoise":{"available":true,"value":18.775},"difficulty":{"available":true,"value":168.7}},{"slug":"dual-exam-agent-index","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"sci-agent-exam","signalToNoise":{"available":true,"value":7.509},"difficulty":{"available":true,"value":153.2}},{"slug":"swe-shift-shifted","signalToNoise":{"available":true,"value":8.943},"difficulty":{"available":true,"value":189.4}},{"slug":"swe-shift-clean","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"crm-agent-bench","signalToNoise":{"available":true,"value":4.527},"difficulty":{"available":true,"value":101.0}},{"slug":"tampered-ml","signalToNoise":{"available":true,"value":8.738},"difficulty":{"available":true,"value":156.9}},{"slug":"agent4sci-mean-relative-improvement","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"regretbench-success","signalToNoise":{"available":true,"value":1.754},"difficulty":{"available":true,"value":101.6}},{"slug":"regretbench-normalized-regret","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vulnpr-100","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"code2video-aa","signalToNoise":{"available":true,"value":12.923},"difficulty":{"available":true,"value":160.7}},{"slug":"exploitbench-internal-port-june-august-2026","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"secprbench","signalToNoise":{"available":true,"value":5.565},"difficulty":{"available":true,"value":152.6}},{"slug":"benchbench-cloning","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"continual-learning-exam","signalToNoise":{"available":true,"value":9.246},"difficulty":{"available":true,"value":176.4}},{"slug":"continual-learning-exam-trial-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"delta-tau-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"bizagentbench","signalToNoise":{"available":true,"value":14.728},"difficulty":{"available":true,"value":161.9}},{"slug":"quant-arena","signalToNoise":{"available":true,"value":3.118},"difficulty":{"available":true,"value":177.8}},{"slug":"p2npcbench","signalToNoise":{"available":true,"value":1.725},"difficulty":{"available":true,"value":163.6}},{"slug":"p2npcbench-p","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-progress-index-v1","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"dayjob-tax","signalToNoise":{"available":true,"value":18.027},"difficulty":{"available":true,"value":184.2}},{"slug":"frontiercode-2-0-score-cognition","signalToNoise":{"available":true,"value":17.36},"difficulty":{"available":true,"value":170.7}},{"slug":"frontiercode-2-0-pass-rate-cognition","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"embodiedlab","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-fine-manipulation","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-long-horizon","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-fine-reading","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-multi-point-data","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-basic","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-advanced","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-process-score","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-t1","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"embodiedlab-t2","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"iprefbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"iprefbench-action-completion","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"iprefbench-personalization","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"iprefbench-ask-recall","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"iprefbench-ask-precision","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vibevideobench","signalToNoise":{"available":true,"value":4.163},"difficulty":{"available":true,"value":145.4}},{"slug":"vibevideobench-strict-success","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vibevideobench-spec-compliance","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"featurearc","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"featurearc-first-stage","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"unitybench-mini3d","signalToNoise":{"available":true,"value":3.181},"difficulty":{"available":true,"value":153.0}},{"slug":"unitybench-mini3d-easy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"unitybench-mini3d-medium","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"unitybench-mini3d-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-intelligence-index-v5-0","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"frontiercode-2-0-aa","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-omniscience-v1-1-index-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aa-omniscience-v1-1-accuracy-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aa-omniscience-v1-1-hallucination-rate-artificial-analysis","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"pwnbench-v0-1","signalToNoise":{"available":true,"value":6.503},"difficulty":{"available":true,"value":159.1}},{"slug":"cybergym-pass3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hlab-contracts","signalToNoise":{"available":true,"value":6.648},"difficulty":{"available":true,"value":199.4}},{"slug":"epiq-corporatebench","signalToNoise":{"available":true,"value":5.928},"difficulty":{"available":true,"value":126.0}},{"slug":"apex-agents-1-1-corporate-law-68","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"big-finance-bench-private-holdout","signalToNoise":{"available":true,"value":5.74},"difficulty":{"available":true,"value":166.7}},{"slug":"hebbia-financial-services-benchmark","signalToNoise":{"available":true,"value":8.562},"difficulty":{"available":true,"value":156.6}},{"slug":"atlas-finance","signalToNoise":{"available":true,"value":19.496},"difficulty":{"available":true,"value":180.3}},{"slug":"sheetbench-50","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"hlab-v1-1-fireworks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"atlas-exa-200","signalToNoise":{"available":true,"value":5.888},"difficulty":{"available":true,"value":167.4}},{"slug":"nimble-web-context","signalToNoise":{"available":true,"value":3.821},"difficulty":{"available":true,"value":128.3}},{"slug":"tau3-banking-fireworks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"duetbench-diagnosis-elo","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cresta-bench","signalToNoise":{"available":true,"value":2.175},"difficulty":{"available":true,"value":147.1}},{"slug":"legacy-bench-fireworks","signalToNoise":{"available":true,"value":2.658},"difficulty":{"available":true,"value":152.7}},{"slug":"deepswe-1-1-fireworks","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vibench-2-0","signalToNoise":{"available":true,"value":14.612},"difficulty":{"available":true,"value":168.7}},{"slug":"qodo-code-review-benchmark-125","signalToNoise":{"available":true,"value":3.689},"difficulty":{"available":true,"value":166.3}},{"slug":"orca-bench-hard-rca-fireworks","signalToNoise":{"available":true,"value":10.897},"difficulty":{"available":true,"value":168.7}},{"slug":"bedside-bench","signalToNoise":{"available":true,"value":6.025},"difficulty":{"available":true,"value":123.1}},{"slug":"cad-arena","signalToNoise":{"available":true,"value":9.687},"difficulty":{"available":true,"value":164.0}},{"slug":"aim-routing","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aim-grounding","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aim-consistency","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"build-a-chip-bench-20-tasks","signalToNoise":{"available":true,"value":8.209},"difficulty":{"available":true,"value":183.8}},{"slug":"odysseys2","signalToNoise":{"available":true,"value":2.446},"difficulty":{"available":true,"value":162.1}},{"slug":"odysseys2-perfect-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"rwe-bench","signalToNoise":{"available":true,"value":56.532},"difficulty":{"available":true,"value":174.2}},{"slug":"rwe-bench-pass-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierml","signalToNoise":{"available":true,"value":15.098},"difficulty":{"available":true,"value":168.4}},{"slug":"frontierml-wallet-50","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierml-wallet-10","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"atlas-finance-1-1","signalToNoise":{"available":true,"value":37.4},"difficulty":{"available":true,"value":178.7}},{"slug":"atlas-finance-1-1-pass-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"atlas-finance-1-1-rubric-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"vials","signalToNoise":{"available":true,"value":4.878},"difficulty":{"available":true,"value":175.2}},{"slug":"bankertoolbench","signalToNoise":{"available":true,"value":10.22},"difficulty":{"available":true,"value":153.3}},{"slug":"bankertoolbench-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"bankertoolbench-pass-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"bankertoolbench-1-1","signalToNoise":{"available":true,"value":24.239},"difficulty":{"available":true,"value":168.5}},{"slug":"bankertoolbench-1-1-pass-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"carebench","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"studentbench-lesson-planning","signalToNoise":{"available":true,"value":2.988},"difficulty":{"available":true,"value":155.4}},{"slug":"studentbench-practice-design","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"studentbench-learning-gain","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"web-clone-bench","signalToNoise":{"available":true,"value":47.577},"difficulty":{"available":true,"value":170.4}},{"slug":"westworld-finance-diligence","signalToNoise":{"available":true,"value":7.597},"difficulty":{"available":true,"value":165.6}},{"slug":"westworld-finance-diligence-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-lcr-mercor-extended","signalToNoise":{"available":true,"value":7.346},"difficulty":{"available":true,"value":154.2}},{"slug":"browsecomp-mercor-extended","signalToNoise":{"available":true,"value":10.52},"difficulty":{"available":true,"value":165.4}},{"slug":"charxiv-mercor-extended","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gdpval-mercor-extended-rubric","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hle-mercor-extended","signalToNoise":{"available":true,"value":11.504},"difficulty":{"available":true,"value":162.3}},{"slug":"medxpertqa-mm-mercor-extended","signalToNoise":{"available":true,"value":4.471},"difficulty":{"available":true,"value":156.9}},{"slug":"mmmu-pro-mercor-extended","signalToNoise":{"available":true,"value":11.73},"difficulty":{"available":true,"value":170.4}},{"slug":"swe-atlas-codebase-qna-mercor-extended","signalToNoise":{"available":true,"value":13.127},"difficulty":{"available":true,"value":168.8}},{"slug":"swe-bench-verified-mercor-extended","signalToNoise":{"available":true,"value":4.784},"difficulty":{"available":true,"value":158.6}},{"slug":"terminal-bench-2-1-mercor-extended","signalToNoise":{"available":true,"value":4.122},"difficulty":{"available":true,"value":174.3}},{"slug":"frontierphysics-0-1","signalToNoise":{"available":true,"value":33.409},"difficulty":{"available":true,"value":174.5}},{"slug":"frontierphysics-0-1-verifier-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierphysics-0-1-pass-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontierphysics-0-1-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"witnessbench-2-0","signalToNoise":{"available":true,"value":9.23},"difficulty":{"available":true,"value":175.9}},{"slug":"matflowbench","signalToNoise":{"available":true,"value":27.571},"difficulty":{"available":true,"value":166.2}},{"slug":"matflowbench-pass-rate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"matflowbench-computational","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"matflowbench-experimental","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"matflowbench-computational-pass-at-2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"matflowbench-experimental-pass-at-4","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"frontiercua","signalToNoise":{"available":true,"value":9.779},"difficulty":{"available":true,"value":171.8}},{"slug":"frontiercua-avg-at-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"autoresearchexam","signalToNoise":{"available":true,"value":11.983},"difficulty":{"available":true,"value":162.3}},{"slug":"aml-bench","signalToNoise":{"available":true,"value":4.305},"difficulty":{"available":true,"value":135.8}},{"slug":"aml-bench-pass-hat-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"treasuryqa-bench","signalToNoise":{"available":true,"value":4.86},"difficulty":{"available":true,"value":168.7}},{"slug":"treasuryqa-bench-pass-hat-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sage-v1-1","signalToNoise":{"available":true,"value":9.29},"difficulty":{"available":true,"value":144.1}},{"slug":"sage-v1-1-pdes","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sage-v1-1-linear-algebra","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sage-v1-1-cs","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"immigration-bench","signalToNoise":{"available":true,"value":8.147},"difficulty":{"available":true,"value":142.6}},{"slug":"immigration-bench-all-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"immigration-bench-intake","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"immigration-bench-content","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cyber-offense","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cyber-offense-nova","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cyber-offense-lighthouse","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"yc-bench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"yc-bench-bankruptcy-rate","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hle-diamond-reasoning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hle-diamond-knowledge","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"interactivebench","signalToNoise":{"available":true,"value":8.531},"difficulty":{"available":true,"value":175.5}},{"slug":"interactivebench-guidance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"interactivebench-planning","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"interactivebench-accessibility","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"interactivebench-decision","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"interactivebench-prevention","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"interactivebench-finding","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gym-anything-rt","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gym-anything-rt-non-real-time","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gym-anything-rt-paused","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"barc-bench","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"dayjob-law","signalToNoise":{"available":true,"value":5.309},"difficulty":{"available":true,"value":181.0}},{"slug":"dayjob-accounting","signalToNoise":{"available":true,"value":11.903},"difficulty":{"available":true,"value":174.8}},{"slug":"dayjob-data-science","signalToNoise":{"available":true,"value":40.924},"difficulty":{"available":true,"value":169.9}},{"slug":"warc-bench-test-192","signalToNoise":{"available":true,"value":3.079},"difficulty":{"available":true,"value":143.2}},{"slug":"swe-clar-implementation","signalToNoise":{"available":true,"value":7.982},"difficulty":{"available":true,"value":158.6}},{"slug":"swe-clar-clarification","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ramp-accounting-bench","signalToNoise":{"available":true,"value":8.792},"difficulty":{"available":true,"value":166.9}},{"slug":"ramp-accounting-bench-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"dualbench100","signalToNoise":{"available":true,"value":4.023},"difficulty":{"available":true,"value":120.9}},{"slug":"dualbench100-consistency","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aa-finance-accounting-index","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"aa-omniscience-business-accuracy","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aa-omniscience-business-non-hallucination","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"automationbench-aa-finance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hedge-bench","signalToNoise":{"available":true,"value":13.22},"difficulty":{"available":true,"value":171.1}},{"slug":"hedge-bench-coverage","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hedge-bench-forecast-accuracy","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"investors-last-exam","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"investors-last-exam-strict","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"frontier-financial-judgement","signalToNoise":{"available":true,"value":9.871},"difficulty":{"available":true,"value":160.1}},{"slug":"frontier-financial-judgement-atomic","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"xbench-scienceqa","signalToNoise":{"available":true,"value":4.685},"difficulty":{"available":true,"value":117.5}},{"slug":"xbench-scienceqa-pass-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sab-research","signalToNoise":{"available":true,"value":9.192},"difficulty":{"available":true,"value":124.3}},{"slug":"sab-research-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sab-research-withheld-answered","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"sab-ranking","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"dual-arm-sim","signalToNoise":{"available":true,"value":9.738},"difficulty":{"available":true,"value":194.2}},{"slug":"dual-arm-sim-progress","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"dual-arm-sim-instrument-swap","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"dual-arm-sim-cartridge-swap","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"dual-arm-sim-sample-transfer","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"dual-arm-sim-parts-kitting","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"dual-arm-sim-false-completion-claims","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"mbpp","signalToNoise":{"available":true,"value":2.506},"difficulty":{"available":true,"value":95.6}},{"slug":"natural-questions","signalToNoise":{"available":true,"value":2.331},"difficulty":{"available":true,"value":132.3}},{"slug":"siqa","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"siqa-olmo3-mc","signalToNoise":{"available":true,"value":7.445},"difficulty":{"available":true,"value":79.9}},{"slug":"drop","signalToNoise":{"available":true,"value":2.856},"difficulty":{"available":true,"value":82.7}},{"slug":"docvqa","signalToNoise":{"available":true,"value":2.17},"difficulty":{"available":true,"value":43.9}},{"slug":"infovqa","signalToNoise":{"available":true,"value":1.472},"difficulty":{"available":true,"value":82.1}},{"slug":"chartqa-test","signalToNoise":{"available":true,"value":1.384},"difficulty":{"available":true,"value":63.4}},{"slug":"ai2d","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"medxpertqa-text","signalToNoise":{"available":true,"value":6.923},"difficulty":{"available":true,"value":153.9}},{"slug":"medxpertqa-mm","signalToNoise":{"available":true,"value":9.719},"difficulty":{"available":true,"value":147.8}},{"slug":"medmcqa","signalToNoise":{"available":true,"value":4.123},"difficulty":{"available":true,"value":125.6}},{"slug":"pubmedqa","signalToNoise":{"available":true,"value":2.857},"difficulty":{"available":true,"value":100.2}},{"slug":"mt-bench","signalToNoise":{"available":true,"value":8.378},"difficulty":{"available":true,"value":76.3}},{"slug":"mt-bench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mm-mt-bench","signalToNoise":{"available":true,"value":3.64},"difficulty":{"available":true,"value":108.5}},{"slug":"arena-hard-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arena-hard-v2-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"wildbench-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"wildbench-v2-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gsm8k-maj1-8","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"math-maj1-4","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"gpqa-main","signalToNoise":{"available":true,"value":8.118},"difficulty":{"available":true,"value":137.7}},{"slug":"vqav2-val","signalToNoise":{"available":true,"value":2.117},"difficulty":{"available":true,"value":82.6}},{"slug":"mmlu-redux","signalToNoise":{"available":true,"value":9.158},"difficulty":{"available":true,"value":103.7}},{"slug":"supergpqa","signalToNoise":{"available":true,"value":12.655},"difficulty":{"available":true,"value":138.5}},{"slug":"c-eval","signalToNoise":{"available":true,"value":2.311},"difficulty":{"available":true,"value":103.8}},{"slug":"longbench-v2","signalToNoise":{"available":true,"value":6.955},"difficulty":{"available":true,"value":152.3}},{"slug":"imo-answerbench","signalToNoise":{"available":true,"value":5.981},"difficulty":{"available":true,"value":131.5}},{"slug":"imo-proofbench-advanced-usamo-2025","signalToNoise":{"available":true,"value":7.978},"difficulty":{"available":true,"value":154.7}},{"slug":"mcpmark","signalToNoise":{"available":true,"value":14.07},"difficulty":{"available":true,"value":152.7}},{"slug":"browsecomp-zh","signalToNoise":{"available":true,"value":8.551},"difficulty":{"available":true,"value":144.3}},{"slug":"seal-0","signalToNoise":{"available":true,"value":3.808},"difficulty":{"available":true,"value":156.2}},{"slug":"mmlu-prox","signalToNoise":{"available":true,"value":9.563},"difficulty":{"available":true,"value":120.8}},{"slug":"include","signalToNoise":{"available":true,"value":10.388},"difficulty":{"available":true,"value":126.7}},{"slug":"polymath","signalToNoise":{"available":true,"value":5.95},"difficulty":{"available":true,"value":141.1}},{"slug":"nova-63","signalToNoise":{"available":true,"value":3.643},"difficulty":{"available":true,"value":131.8}},{"slug":"deepplanning","signalToNoise":{"available":true,"value":7.476},"difficulty":{"available":true,"value":157.5}},{"slug":"tau2-bench-average","signalToNoise":{"available":true,"value":4.981},"difficulty":{"available":true,"value":128.7}},{"slug":"mmstar","signalToNoise":{"available":true,"value":5.894},"difficulty":{"available":true,"value":121.9}},{"slug":"hallusionbench","signalToNoise":{"available":true,"value":3.713},"difficulty":{"available":true,"value":112.7}},{"slug":"mmbench-v1-1-dev-en","signalToNoise":{"available":true,"value":4.827},"difficulty":{"available":true,"value":86.3}},{"slug":"simplevqa","signalToNoise":{"available":true,"value":3.289},"difficulty":{"available":true,"value":133.6}},{"slug":"ocrbench","signalToNoise":{"available":true,"value":1.661},"difficulty":{"available":true,"value":37.3}},{"slug":"cc-ocr","signalToNoise":{"available":true,"value":1.815},"difficulty":{"available":true,"value":81.4}},{"slug":"mmlongbench-doc","signalToNoise":{"available":true,"value":3.827},"difficulty":{"available":true,"value":139.8}},{"slug":"countbench","signalToNoise":{"available":true,"value":3.923},"difficulty":{"available":true,"value":85.8}},{"slug":"dynamath","signalToNoise":{"available":true,"value":6.679},"difficulty":{"available":true,"value":107.2}},{"slug":"we-math","signalToNoise":{"available":true,"value":4.796},"difficulty":{"available":true,"value":129.8}},{"slug":"lingoqa","signalToNoise":{"available":true,"value":1.732},"difficulty":{"available":true,"value":131.3}},{"slug":"vstar-bench","signalToNoise":{"available":true,"value":1.389},"difficulty":{"available":true,"value":91.7}},{"slug":"vstar-bench-with-python","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"video-mmmu","signalToNoise":{"available":true,"value":4.967},"difficulty":{"available":true,"value":120.1}},{"slug":"mvbench","signalToNoise":{"available":true,"value":1.844},"difficulty":{"available":true,"value":89.9}},{"slug":"mmvu","signalToNoise":{"available":true,"value":4.517},"difficulty":{"available":true,"value":118.3}},{"slug":"slake","signalToNoise":{"available":true,"value":2.755},"difficulty":{"available":true,"value":97.1}},{"slug":"pmc-vqa","signalToNoise":{"available":true,"value":2.393},"difficulty":{"available":true,"value":157.6}},{"slug":"vlmsareblind","signalToNoise":{"available":true,"value":2.309},"difficulty":{"available":true,"value":100.8}},{"slug":"videomme-with-subtitles","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arena-hard-v2","signalToNoise":{"available":true,"value":6.923},"difficulty":{"available":true,"value":135.1}},{"slug":"alignbench-v1-1","signalToNoise":{"available":true,"value":3.529},"difficulty":{"available":true,"value":91.9}},{"slug":"writingbench","signalToNoise":{"available":true,"value":2.761},"difficulty":{"available":true,"value":104.4}},{"slug":"zebralogic","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"multi-if","signalToNoise":{"available":true,"value":2.574},"difficulty":{"available":true,"value":112.0}},{"slug":"bfcl-v3-overall-accuracy","signalToNoise":{"available":true,"value":2.767},"difficulty":{"available":true,"value":113.2}},{"slug":"tau-bench-retail","signalToNoise":{"available":true,"value":5.471},"difficulty":{"available":true,"value":129.5}},{"slug":"tau-bench-airline","signalToNoise":{"available":true,"value":4.644},"difficulty":{"available":true,"value":141.7}},{"slug":"livecodebench-2025-02-01-2025-05-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"hmmt-feb-2025","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmbench-v1-1-en","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mathverse-mini","signalToNoise":{"available":true,"value":4.696},"difficulty":{"available":true,"value":118.9}},{"slug":"logicvista","signalToNoise":{"available":true,"value":6.21},"difficulty":{"available":true,"value":125.8}},{"slug":"blink","signalToNoise":{"available":true,"value":2.819},"difficulty":{"available":true,"value":107.2}},{"slug":"muirbench","signalToNoise":{"available":true,"value":3.881},"difficulty":{"available":true,"value":108.7}},{"slug":"mlvu-m-avg","signalToNoise":{"available":true,"value":2.333},"difficulty":{"available":true,"value":97.7}},{"slug":"cmmlu","signalToNoise":{"available":true,"value":2.771},"difficulty":{"available":true,"value":99.4}},{"slug":"humaneval-plus","signalToNoise":{"available":true,"value":1.646},"difficulty":{"available":true,"value":100.7}},{"slug":"theoremqa","signalToNoise":{"available":true,"value":3.861},"difficulty":{"available":true,"value":133.2}},{"slug":"mmlu-stem","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"truthfulqa-unspecified-metric","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"gpqa-reported-subset-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arena-hard-v0-1","signalToNoise":{"available":true,"value":6.789},"difficulty":{"available":true,"value":115.5}},{"slug":"multipl-e-humaneval-8-language-average","signalToNoise":{"available":true,"value":2.002},"difficulty":{"available":true,"value":102.4}},{"slug":"live-bench-2024-08-31","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-v1","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2023-05-01-2024-09-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ifeval-prompt-level-strict","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"livecodebench-2024-08-01-2024-11-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2024-08-01-2025-02-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ruler","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-4k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-8k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-16k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-32k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-64k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-128k","signalToNoise":{"available":true,"value":1.284},"difficulty":{"available":true,"value":105.1}},{"slug":"livecodebench-2024-07-01-2024-09-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2024-07-01-2024-11-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"textvqa-val","signalToNoise":{"available":true,"value":3.366},"difficulty":{"available":true,"value":51.7}},{"slug":"mmbench-en-test","signalToNoise":{"available":true,"value":5.842},"difficulty":{"available":true,"value":72.7}},{"slug":"mmbench-cn-test","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmbench-v1-1-en-test","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hallusionbench-avg","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"mmvet","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"screenspot","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"facts-grounding","signalToNoise":{"available":true,"value":1.698},"difficulty":{"available":true,"value":80.0}},{"slug":"vibe-eval-gemini-judge","signalToNoise":{"available":true,"value":8.543},"difficulty":{"available":true,"value":124.4}},{"slug":"egoschema","signalToNoise":{"available":true,"value":2.893},"difficulty":{"available":true,"value":113.3}},{"slug":"bird-sql-dev","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vqav2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"activitynet-qa","signalToNoise":{"available":true,"value":2.598},"difficulty":{"available":true,"value":113.7}},{"slug":"perception-test-mcqa","signalToNoise":{"available":true,"value":3.6},"difficulty":{"available":true,"value":122.9}},{"slug":"simpleqa-f1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"quality","signalToNoise":{"available":true,"value":13.3},"difficulty":{"available":true,"value":91.1}},{"slug":"terminal-bench-1-0","signalToNoise":{"available":true,"value":4.558},"difficulty":{"available":true,"value":151.7}},{"slug":"wmdp-bio","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"wmdp-chem","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"wmdp-cyber","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"vct","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"biolp-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lab-bench-protocolqa","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lab-bench-figqa","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"lab-bench-cloningscenarios","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"webarena-verified","signalToNoise":{"available":true,"value":7.244},"difficulty":{"available":true,"value":151.2}},{"slug":"simplevqa-with-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"commonsenseqa","signalToNoise":{"available":true,"value":4.145},"difficulty":{"available":true,"value":89.6}},{"slug":"squad2","signalToNoise":{"available":true,"value":2.446},"difficulty":{"available":true,"value":66.4}},{"slug":"quac","signalToNoise":{"available":true,"value":2.058},"difficulty":{"available":true,"value":115.5}},{"slug":"mbpp-evalplus-base","signalToNoise":{"available":true,"value":9.435},"difficulty":{"available":true,"value":88.9}},{"slug":"api-bank-meta-corrected","signalToNoise":{"available":true,"value":4.88},"difficulty":{"available":true,"value":98.2}},{"slug":"gorilla-apibench","signalToNoise":{"available":true,"value":7.012},"difficulty":{"available":true,"value":132.1}},{"slug":"nexus","signalToNoise":{"available":true,"value":3.603},"difficulty":{"available":true,"value":124.2}},{"slug":"bfcl-v2-ast-summary","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"infinitebench-en-mc","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"infinitebench-en-qa","signalToNoise":{"available":true,"value":1.337},"difficulty":{"available":true,"value":170.0}},{"slug":"nih-multi-needle-meta","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mmmu-pro-vision","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"trivia-qa-wiki-dev","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"trivia-qa-filtered","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"race-middle","signalToNoise":{"available":true,"value":3.136},"difficulty":{"available":true,"value":89.2}},{"slug":"race-high","signalToNoise":{"available":true,"value":1.559},"difficulty":{"available":true,"value":108.2}},{"slug":"humaneval-pass10","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"humaneval-pass100","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mbpp-pass10","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mbpp-pass80","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mbpp-pass100","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"truthfulqa-open-truth-info-percent","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"truthfulqa-open-truth-percent","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"truthfulqa-percent-informative","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"toxigen-toxic-generation-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"multipl-e-humaneval-cpp","signalToNoise":{"available":true,"value":2.9},"difficulty":{"available":true,"value":104.0}},{"slug":"multipl-e-humaneval-java","signalToNoise":{"available":true,"value":3.114},"difficulty":{"available":true,"value":102.9}},{"slug":"multipl-e-humaneval-php","signalToNoise":{"available":true,"value":3.478},"difficulty":{"available":true,"value":105.6}},{"slug":"multipl-e-humaneval-typescript","signalToNoise":{"available":true,"value":2.479},"difficulty":{"available":true,"value":109.0}},{"slug":"multipl-e-humaneval-csharp","signalToNoise":{"available":true,"value":1.145},"difficulty":{"available":true,"value":98.7}},{"slug":"multipl-e-humaneval-bash","signalToNoise":{"available":true,"value":3.716},"difficulty":{"available":true,"value":124.0}},{"slug":"cmrc-2018","signalToNoise":{"available":true,"value":4.966},"difficulty":{"available":true,"value":80.7}},{"slug":"c3","signalToNoise":{"available":true,"value":3.742},"difficulty":{"available":true,"value":57.6}},{"slug":"ccpm","signalToNoise":{"available":true,"value":1.813},"difficulty":{"available":true,"value":76.9}},{"slug":"chid","signalToNoise":{"available":true,"value":1.314},"difficulty":{"available":true,"value":63.6}},{"slug":"cluewsc","signalToNoise":{"available":true,"value":3.663},"difficulty":{"available":true,"value":95.2}},{"slug":"cmath","signalToNoise":{"available":true,"value":2.374},"difficulty":{"available":true,"value":80.1}},{"slug":"cruxeval-i","signalToNoise":{"available":true,"value":7.818},"difficulty":{"available":true,"value":110.9}},{"slug":"cruxeval-o","signalToNoise":{"available":true,"value":5.989},"difficulty":{"available":true,"value":109.5}},{"slug":"humaneval-mul","signalToNoise":{"available":true,"value":1.626},"difficulty":{"available":true,"value":84.1}},{"slug":"alignbench","signalToNoise":{"available":true,"value":6.159},"difficulty":{"available":true,"value":94.0}},{"slug":"frames","signalToNoise":{"available":true,"value":3.027},"difficulty":{"available":true,"value":99.7}},{"slug":"chinese-simpleqa","signalToNoise":{"available":true,"value":2.598},"difficulty":{"available":true,"value":117.3}},{"slug":"aider-code-editing","signalToNoise":{"available":true,"value":11.279},"difficulty":{"available":true,"value":109.2}},{"slug":"agieval","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"codeforces-percentile-deepseek-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"cnmo-2024","signalToNoise":{"available":true,"value":5.369},"difficulty":{"available":true,"value":142.6}},{"slug":"codeforces-rating-deepseek-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2024-08-01-2025-05-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"codeforces-div1-rating-deepseek-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"simpleqa-with-search","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mcp-universe","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"codeforces-div1-2025-deepseek-v4-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"corpusqa-1m","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mcp-atlas-deepseek-v4-protocol-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2023-09-01-2024-04-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2023-12-01-2024-06-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"livecodebench-2024-01-01-2024-09-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mt-bench-gpt-4-turbo","signalToNoise":{"available":true,"value":5.548},"difficulty":{"available":true,"value":77.7}},{"slug":"bfcl-v2-live","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ifeval-instruction-strict","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"multi-swe-bench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"ojbench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ojbench-cpp","signalToNoise":{"available":true,"value":11.706},"difficulty":{"available":true,"value":149.8}},{"slug":"ojbench-python","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"paperbench-code-dev","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mcpmark-verified","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gsm8k-platinum","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cruxeval-i-cot","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cruxeval-o-cot","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"evalplus","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"livecodebench-2024-12-01-2025-02-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"screenspot-v2","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"longvideobench-reported","signalToNoise":{"available":true,"value":3.039},"difficulty":{"available":true,"value":110.3}},{"slug":"livecodebench-2024-07-01-2025-01-01","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ruler-256k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-512k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ruler-1m","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arena-hard-v2-hard-prompts","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arena-hard-v2-creative-writing","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"arena-hard-v2-average","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"collie","signalToNoise":{"available":true,"value":5.581},"difficulty":{"available":true,"value":123.9}},{"slug":"openai-mrcr-2-needle-128k","signalToNoise":{"available":true,"value":1.978},"difficulty":{"available":true,"value":138.7}},{"slug":"openai-mrcr-2-needle-256k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"graphwalks-parents-up-to-128k","signalToNoise":{"available":true,"value":4.753},"difficulty":{"available":true,"value":137.5}},{"slug":"browsecomp-long-context-128k","signalToNoise":{"available":true,"value":2.676},"difficulty":{"available":true,"value":75.2}},{"slug":"browsecomp-long-context-256k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"videomme-long-with-subtitles","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"swe-lancer-ic-swe-diamond","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"multichallenge-o3-mini-grader-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-api-instruction-following-hard-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"longfact-concepts-hallucination-rate-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"longfact-objects-hallucination-rate-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"factscore-hallucination-rate-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"frontiermath-tiers-1-3-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-code-refactoring-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"swe-lancer-ic-swe-pass-1-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-professional-ctf-pass-12-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-prs-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mle-bench-30-bronze-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"paperbench-10-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-proof-qa-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"osworld-2-0808-offline-reported-partial","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"codeforces-elo-openai-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"openai-mrcr-2-needle-1m","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"videomme-long-no-subtitles","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"complexfuncbench","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"claw-eval-general-en-pass-hat-3","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"claw-eval-general-en-average-3","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"enterpriseops-gym","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"automationbench-public-set-metric-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"toolsandbox","signalToNoise":{"available":true,"value":2.819},"difficulty":{"available":true,"value":95.5}},{"slug":"mcp-atlas-lfm3-protocol-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"ifeval-reported","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ifstruct-v1-0","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"hle-with-tools-lfm3-cohort-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-omniscience-public-index","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aa-omniscience-public-accuracy","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aa-omniscience-public-non-hallucination","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"daily-omni","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"worldsense","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"av-speakerbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"omnibench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"omnibench-speech","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"open-asr-english-average-wer-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"seed-tts-eval-en-utmosv2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"voicebench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mmar","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"mme","signalToNoise":{"available":true,"value":2.539},"difficulty":{"available":true,"value":101.4}},{"slug":"simplevqa-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"seed-bench-image","signalToNoise":{"available":true,"value":2.773},"difficulty":{"available":true,"value":65.6}},{"slug":"mm-ifeval","signalToNoise":{"available":true,"value":2.629},"difficulty":{"available":true,"value":107.7}},{"slug":"countbenchqa","signalToNoise":{"available":true,"value":3.27},"difficulty":{"available":true,"value":75.1}},{"slug":"pope-f1","signalToNoise":{"available":true,"value":1.977},"difficulty":{"available":true,"value":22.5}},{"slug":"perceptionbench-gpt-6-luna-judge","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"hr-bench-8k","signalToNoise":{"available":true,"value":1.829},"difficulty":{"available":true,"value":116.1}},{"slug":"hr-bench-4k","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"cv-bench","signalToNoise":{"available":true,"value":1.986},"difficulty":{"available":true,"value":90.8}},{"slug":"docvqa-val","signalToNoise":{"available":true,"value":5.88},"difficulty":{"available":true,"value":62.9}},{"slug":"ocrbench-v2-en","signalToNoise":{"available":true,"value":4.283},"difficulty":{"available":true,"value":117.9}},{"slug":"infographicvqa-val","signalToNoise":{"available":true,"value":5.522},"difficulty":{"available":true,"value":91.7}},{"slug":"we-math-testmini-strict","signalToNoise":{"available":true,"value":9.789},"difficulty":{"available":true,"value":127.6}},{"slug":"refcoco-8-split-average","signalToNoise":{"available":true,"value":1.867},"difficulty":{"available":true,"value":77.5}},{"slug":"refcoco-m-reported","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"point-bench","signalToNoise":{"available":true,"value":1.109},"difficulty":{"available":true,"value":123.2}},{"slug":"refspatial-bench","signalToNoise":{"available":true,"value":0.808},"difficulty":{"available":true,"value":156.2}},{"slug":"embspatial-bench","signalToNoise":{"available":true,"value":1.973},"difficulty":{"available":true,"value":100.6}},{"slug":"robospatial-home","signalToNoise":{"available":true,"value":1.688},"difficulty":{"available":true,"value":121.1}},{"slug":"where2place","signalToNoise":{"available":true,"value":1.598},"difficulty":{"available":true,"value":150.2}},{"slug":"openeqa-em-eqa","signalToNoise":{"available":true,"value":2.16},"difficulty":{"available":true,"value":111.3}},{"slug":"vsi-bench","signalToNoise":{"available":true,"value":1.049},"difficulty":{"available":true,"value":147.8}},{"slug":"screenspot-v2-desktop","signalToNoise":{"available":true,"value":2.222},"difficulty":{"available":true,"value":92.1}},{"slug":"screenspot-v2-mobile","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"screenspot-v2-web","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"motionbench-dev","signalToNoise":{"available":true,"value":2.496},"difficulty":{"available":true,"value":146.1}},{"slug":"mm-toolsandbox-partial-credit-lfm3","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"tau3-banking-knowledge-bm25","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"casereportbench-metric-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mmmlu-ar-de-es-fr-ja-ko-zh","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"librispeech-test-clean","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"librispeech-test-other","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"ami-esb-test","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"earnings22-esb-test","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"gigaspeech-esb-test","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"spgispeech-esb-test","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"tedlium-3-esb-test","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"voxpopuli-en-esb-test","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"wildclawbench","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gaia2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"pinchbench-meta-container","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"mcp-atlas-muse-glimmer-public-500","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"skillsbench-with-skills-86-weighted-tests","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"omnidocbench-v1-5-meta-two-component","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"beam-128k","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"cimemories-violation-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"cimemories-coverage-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"privacylens-adjusted-leakage-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"siren-agentdojo-attack-success-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"siren-agentdojo-utility-under-attack","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"omnilingual-gaia2","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"linguini","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"belebele","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"wmt24pp-chrf","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"flores-plus-devtest-chrf","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"bouquet-chrf","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"curator-evals-code-correctness","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"macroscopebench","signalToNoise":{"available":true,"value":5.559},"difficulty":{"available":true,"value":139.6}},{"slug":"macroscopebench-recall","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"macroscopebench-precision","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"apex-accounting-1-1","signalToNoise":{"available":true,"value":7.683},"difficulty":{"available":true,"value":158.5}},{"slug":"legalreasonbench","signalToNoise":{"available":true,"value":4.824},"difficulty":{"available":true,"value":172.5}},{"slug":"realm-financial","signalToNoise":{"available":true,"value":7.845},"difficulty":{"available":true,"value":167.8}},{"slug":"pharmabench-micro1","signalToNoise":{"available":true,"value":5.715},"difficulty":{"available":true,"value":195.8}},{"slug":"pharmabench-micro1-mean-score","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"wellnessbench","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"missionreport","signalToNoise":{"available":true,"value":14.489},"difficulty":{"available":true,"value":153.4}},{"slug":"globebench","signalToNoise":{"available":true,"value":4.717},"difficulty":{"available":true,"value":181.1}},{"slug":"ma-bench","signalToNoise":{"available":true,"value":2.909},"difficulty":{"available":true,"value":158.3}},{"slug":"ide-bench","signalToNoise":{"available":true,"value":5.407},"difficulty":{"available":true,"value":144.4}},{"slug":"ide-bench-pass-at-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"financeqa","signalToNoise":{"available":true,"value":3.214},"difficulty":{"available":true,"value":149.8}},{"slug":"financeqa-basic-tactical","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"financeqa-conceptual","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"witnessbench-1-0","signalToNoise":{"available":true,"value":7.552},"difficulty":{"available":true,"value":177.3}},{"slug":"veracode-genai-code-security","signalToNoise":{"available":true,"value":2.801},"difficulty":{"available":true,"value":135.7}},{"slug":"veracode-genai-code-security-fall-2026","signalToNoise":{"available":true,"value":4.341},"difficulty":{"available":true,"value":141.2}},{"slug":"psebench","signalToNoise":{"available":true,"value":7.527},"difficulty":{"available":true,"value":130.4}},{"slug":"clinicare-bench","signalToNoise":{"available":true,"value":6.364},"difficulty":{"available":true,"value":141.0}},{"slug":"morebench","signalToNoise":{"available":true,"value":3.894},"difficulty":{"available":true,"value":99.1}},{"slug":"aspi-execution-tool-attack-success-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"aspi-clarification-ask-user-attack-success-rate","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"multifiletest-python","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"multifiletest-java","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"multifiletest-javascript","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"gsm1k-lm-eval-harness","signalToNoise":{"available":true,"value":6.698},"difficulty":{"available":true,"value":100.3}},{"slug":"harvey-lab-mercor","signalToNoise":{"available":true,"value":6.592},"difficulty":{"available":true,"value":186.7}},{"slug":"harvey-lab-mercor-subset","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"tau3-banking-mercor","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"toloka-arena-v3","signalToNoise":{"available":true,"value":6.607},"difficulty":{"available":true,"value":160.5}},{"slug":"toloka-arena-v3-pass-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"toloka-arena-v1","signalToNoise":{"available":true,"value":9.031},"difficulty":{"available":true,"value":161.1}},{"slug":"toloka-arena-v1-pass-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"revelo-code-index","signalToNoise":{"available":true,"value":16.074},"difficulty":{"available":true,"value":165.9}},{"slug":"revelo-code-index-swe-lancer","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"revelo-code-index-terminal-bench-2-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"revelo-code-index-terminal-bench-3-0","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"turing-vlmbench","signalToNoise":{"available":true,"value":4.211},"difficulty":{"available":true,"value":159.9}},{"slug":"turing-vlmbench-hard","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"erdosbench","signalToNoise":{"available":true,"value":8.121},"difficulty":{"available":true,"value":146.6}},{"slug":"stembench120-1-0","signalToNoise":{"available":true,"value":4.244},"difficulty":{"available":true,"value":141.0}},{"slug":"stembench120-1-0-with-tools","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"kellybench","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"babaisbench-intro","signalToNoise":{"available":true,"value":5.081},"difficulty":{"available":true,"value":150.4}},{"slug":"babaisbench-intro-pass-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"babaisbench-lake","signalToNoise":{"available":true,"value":13.892},"difficulty":{"available":true,"value":154.8}},{"slug":"compilebench","signalToNoise":{"available":true,"value":5.997},"difficulty":{"available":true,"value":137.3}},{"slug":"compilebench-pass-at-3","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"shroombench","signalToNoise":{"available":true,"value":3.181},"difficulty":{"available":true,"value":165.9}},{"slug":"shroombench-top-5","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"binaryaudit","signalToNoise":{"available":true,"value":1.667},"difficulty":{"available":true,"value":182.1}},{"slug":"binaryaudit-tooling","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"binaryaudit-false-positive-rate","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"otelbench","signalToNoise":{"available":true,"value":9.19},"difficulty":{"available":true,"value":166.0}},{"slug":"gdpx","signalToNoise":{"available":true,"value":32.253},"difficulty":{"available":true,"value":185.1}},{"slug":"gdpx-chip-design","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-cybersecurity","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-biology","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-finance","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-legal","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-quant-trading","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-rsi","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-hardware-design","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-materials-science","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"gdpx-alignment","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"gdpx-software-engineering","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"ifeval-helm-capabilities","signalToNoise":{"available":true,"value":2.262},"difficulty":{"available":true,"value":65.6}},{"slug":"wildbench-helm-capabilities","signalToNoise":{"available":true,"value":5.859},"difficulty":{"available":true,"value":73.1}},{"slug":"omni-math-helm-capabilities","signalToNoise":{"available":true,"value":3.511},"difficulty":{"available":true,"value":138.8}},{"slug":"chartography-surge-ai-2-with-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aime-2024-with-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"aime-2025-with-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"screenspot-pro-with-tools","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"browsecomp-without-browsing","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"arc-challenge-cloze-likelihood","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"lifeval","signalToNoise":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."},"difficulty":{"available":false,"reasonCode":"insufficient_family_models","explanation":"Too few models have been measured on this benchmark to estimate its difficulty."}},{"slug":"matplotgym","signalToNoise":{"available":true,"value":2.479},"difficulty":{"available":true,"value":152.5}},{"slug":"mcp-atlas-longcat-2-5-protocol-unspecified","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-expressiveness","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-organization","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-informativeness","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-transparency","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-warmth","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-conviction","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-imagination","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"aa-personalitybench-slop","signalToNoise":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."},"difficulty":{"available":false,"reasonCode":"diagnostic_benchmark","explanation":"Held out of the fit by design."}},{"slug":"vulcanbench-frontier-v4","signalToNoise":{"available":true,"value":11.997},"difficulty":{"available":true,"value":152.2}},{"slug":"vulcanbench-frontier-v4-pass-at-1","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"llm-chess-elo","signalToNoise":{"available":true,"value":3.486},"difficulty":{"available":true,"value":145.9}},{"slug":"rsi-exam-0-1","signalToNoise":{"available":true,"value":21.156},"difficulty":{"available":true,"value":166.9}},{"slug":"rsi-exam-0-1-public","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"rsi-exam-0-1-private","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"nanogpt-speedrun-frontier","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"nanogpt-speedrun-frontier-24h","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"cheatbench","signalToNoise":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."},"difficulty":{"available":false,"reasonCode":"excluded_benchmark_category","explanation":"This benchmark category is excluded from the capability model and its validation studies."}},{"slug":"drone-bench","signalToNoise":{"available":true,"value":7.365},"difficulty":{"available":true,"value":148.2}},{"slug":"drone-bench-reconstruct","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"drone-bench-localize","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"drone-bench-navigate","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"drone-bench-detect","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"drone-bench-follow","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"tasteval","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"tasteval-performance-multiplier","signalToNoise":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."},"difficulty":{"available":false,"reasonCode":"unbounded","explanation":"This benchmark has no declared score range, so it cannot be placed on the 0-1 scale the capability curve requires."}},{"slug":"long-horizon-terminal-bench-1-0","signalToNoise":{"available":true,"value":7.17},"difficulty":{"available":true,"value":164.4}},{"slug":"long-horizon-terminal-bench-1-0-solved","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"long-horizon-terminal-bench-1-0-strict-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"long-horizon-terminal-bench-2-0","signalToNoise":{"available":true,"value":5.451},"difficulty":{"available":true,"value":156.5}},{"slug":"long-horizon-terminal-bench-2-0-solved","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}},{"slug":"long-horizon-terminal-bench-2-0-strict-pass","signalToNoise":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."},"difficulty":{"available":false,"reasonCode":"not_selected_for_family","explanation":"Another benchmark represents this family in the fit."}}],"preprocessing":{"rows_loaded":148498,"rows_excluded_factor_diagnostic":13485,"rows_excluded_missing_config":3442,"rows_excluded_index_benchmark":4179,"rows_before_deduplication":127392,"rows_excluded_sampling_protocol":35,"multi_row_cells":6747,"cells_resolved_consensus":1650,"cells_collapsed_exact_duplicates":4030,"cells_dropped_conflict":694,"rows_excluded_ambiguous_config_benchmark":2300,"cells_resolved_majority":123,"rows_excluded_majority_outlier":192,"cells_resolved_avg_preference":73,"rows_excluded_avg_preference":122,"cells_resolved_shots_protocol":177,"rows_excluded_shots_protocol":477,"cells_dropped_component_conflict":24,"ambiguous_config_benchmark_cells":694,"rows_collapsed_duplicates":10538,"score_files_skipped_excluded_dirs":0,"rows_retained_after_orientation":113704,"excluded_benchmark_categories":"Safety","rows_excluded_benchmark_category":5941,"families_excluded_benchmark_category":41,"rows_excluded_below_half_chance":26,"rows_excluded_conflict_backfill":489,"rows_excluded_harness_reasoning_model":42,"factor_groups_using_primary":739,"factor_groups_using_components":49,"rows_excluded_primary_preferred":59421,"rows_excluded_primary_undercoverage":744,"rows_excluded_secondary_component":1000,"capability_unit":"model_reasoning_state","reasoning_states_before_filter":2134,"exact_configs_pooled":8688,"rows_collapsed_across_harness_configs":19,"multi_config_state_benchmark_cells":19,"rows_excluded_nonreference_setup":2640,"rows_after_pooling":39963,"rows_admitted_linear_arm":2813,"structural_filter_rounds":3,"linear_arm_cells":2737,"linear_arm_families":39,"linear_arm_models":457,"observations":39391,"benchmark_families":750,"reasoning_states":1917,"models":889,"score_transform":"unit score on the declared range; ratings standardised per family","min_models_per_family":8,"min_families_per_model":4}}