{ "updated": "2026-10-15", "version": "1.2", "sources": [ { "id": "databench", "name": "Hex DataBench v1.1", "url": "https://hex.tech/databench/", "methodologyUrl": "https://hex.tech/blog/databench-agentic-analytics-benchmark/", "date": "2026-09-22", "dateLabel": "Leaderboard updated", "description": "Analytics agents working in a synthetic business warehouse. The published blended score reflects task rubrics, with API cost and median duration measured in the Hex harness.", "limitations": "The v1 design contains 100 tasks. An LLM judge evaluates the work. Version 1.1 changed judge calibration on September 17; older v1 scores are not mixed in. No confidence intervals are published. These costs and times describe this harness, not every agent." }, { "id": "livebench", "name": "LiveBench", "url": "https://livebench.ai/", "methodologyUrl": "https://github.com/LiveBench/new-livebench", "date": null, "dateLabel": "Question-set date shown per study", "description": "Checkable benchmark tasks grouped into reasoning, coding, agentic coding, mathematics, data, language and instruction following. Category means and task-weighted Agent Fit are calculated from the published task scores.", "limitations": "Each sweep uses one question set and one model identity. The question-set date is not a model launch or evaluation date. Unknown default efforts are excluded. Older sweeps do not establish the behavior of newer model versions. No per-mode cost or confidence interval is inferred." }, { "id": "ockbench", "name": "OckBench", "url": "https://ockbench.github.io/", "methodologyUrl": "https://github.com/OckBench/OckBench", "date": null, "dateLabel": "Retrieved September 23, 2026", "description": "200 selected problems: 100 mathematics, 60 coding and 40 science. Accuracy and average output tokens expose quality versus token use; domain results show where effort helps.", "limitations": "Mathematics uses an LLM judge; code runs against tests and science checks answers. Sampling follows model-specific configurations. Opus 5 medium includes nine persistent zero-output failures, counted as incorrect but omitted from average output tokens. Token counts are not dollars or latency." }, { "id": "stet", "name": "Stet coding case study", "url": "https://www.stet.sh/blog/gpt-55-codex-graphql-reasoning-curve", "methodologyUrl": "https://www.stet.sh/blog/gpt-55-codex-graphql-reasoning-curve", "date": "2026-05-07", "dateLabel": "Published", "description": "GPT-5.5 in Codex on 26 matched GraphQL-go-tools repository tasks. Test outcomes, equivalence to human changes and AI review outcomes answer different coding-quality questions.", "limitations": "Exploratory evidence: one repository, one seed per task, stitched runs and uncalibrated GPT-5.4 judging. The publisher calls this inspect-grade, not decision-grade evidence. Xhigh costs retain pre-regrade records. Do not generalize small gaps." }, { "id": "artificial-analysis", "name": "Artificial Analysis", "url": "https://artificialanalysis.ai/models/grok-4-6", "methodologyUrl": "https://artificialanalysis.ai/methodology/intelligence-benchmarking", "date": "2026-09-23", "dateLabel": "Snapshot reviewed", "description": "Grok 4.6 at four explicit effort levels on Intelligence Index v4.3.2. The index combines ten evaluations. Task cost and output tokens use the publisher\u2019s benchmark weights.", "limitations": "Index points are not percent correct or agent success rates. Values retain the precision in the publisher\u2019s public structured data; rounded headline scores can hide small differences. The publisher marks CritPt, one index component, under review. No per-effort uncertainty is provided. A small observed decline does not prove higher effort is worse. Benchmark costs include the source workload and caching; they are not a quote for your agent." }, { "id": "hampson-qwen", "name": "Will Hampson \u00b7 Qwen coding study", "url": "https://whamp.github.io/blog/qwen-3-6-27b-livecodebench-reasoning-budget/", "methodologyUrl": "https://whamp.github.io/blog/qwen-3-6-27b-livecodebench-reasoning-budget/", "date": "2026-05-01", "dateLabel": "Published", "description": "Four matched runs on 50 pinned LiveCodeBench release_v6 LeetCode tasks using public tests. The self-hosted checkpoint is cyankiwi/Qwen3.6-27B-AWQ-BF16-INT4 through vLLM, with temperature 0 and a 65,536-token completion cap.", "limitations": "Exploratory single-setup evidence, not official LiveCodeBench leaderboard scores or hosted Qwen API effort levels. One task changes the score by 2 points. Generation time is a mean on the author\u2019s setup, not a median or universal speed rating. The custom reasoning-end string can affect results. Earlier off and small-budget runs are excluded because the author calls for rerunning them with a corrected tokenizer and matching completion cap. Unrestricted means no reasoning-specific cap; the total completion cap still applies." }, { "id": "joon-codex", "name": "Joonlab \u00b7 Codex subscription study", "url": "https://github.com/joonlab/gpt-6.1-sol-benchmark", "methodologyUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/tree/b38279fd8e28e4534899000d88427af0ddec640b/data", "date": "2026-09-30", "dateLabel": "Measured", "license": "CC BY 4.0", "licenseUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/LICENSE", "description": "Two matched experiments in a ChatGPT Pro Codex environment: 44 math/logic problems repeated twice at medium and high, and one arithmetic prompt repeated three times at five efforts. Data and task definitions are public; the harness and full responses are not.", "limitations": "Exploratory subscription-environment evidence with concurrency 3, not metered API performance. The 44-problem test is near its ceiling and repeats are not independent questions. The five-effort test has only one problem and three runs per cell. Accuracy uncertainty and speed can differ in your workload. Measured September 30; no Ultrafast or current service-tier latency is inferred. Costs are unmeasured. Attribution: Joonlab; CC BY 4.0; rows aggregated and labels adapted by AI Agent Store." } ], "families": [ { "slug": "qwen3-6-27b", "name": "Qwen3.6 27B", "provider": "Alibaba", "profileSlug": "qwen3-6-27b", "studies": [ { "id": "hampson-lcb-50", "source": "hampson-qwen", "name": "Coding token budgets \u00b7 50-task exploratory study", "scope": "Self-hosted AWQ INT4 checkpoint \u00b7 pinned 50 coding tasks \u00b7 65,536 total output cap", "defaultMetric": "overall", "exploratory": true, "control": "token-budget", "durationStatistic": "mean", "resourceCalculator": "generation", "metrics": [ { "key": "overall", "label": "Coding tasks passed", "description": "Percentage of 50 pinned coding tasks passing public functional tests. One task equals 2 points. This is a local experimental slice, not an official LiveCodeBench leaderboard score." } ], "rows": [ { "id": "cyankiwi/Qwen3.6-27B-AWQ-BF16-INT4 / Budget 4096", "effort": "budget-4096", "metrics": { "overall": 82 }, "latency": 115, "outputTokens": 7430, "reasoningTokens": 3537, "budgetTokens": 4096, "completionCap": 65536, "passed": 41, "sampleSize": 50, "completionCapHits": 2, "order": 0 }, { "id": "cyankiwi/Qwen3.6-27B-AWQ-BF16-INT4 / Budget 8192", "effort": "budget-8192", "metrics": { "overall": 86 }, "latency": 131, "outputTokens": 8534, "reasoningTokens": 6046, "budgetTokens": 8192, "completionCap": 65536, "passed": 43, "sampleSize": 50, "completionCapHits": 1, "order": 1 }, { "id": "cyankiwi/Qwen3.6-27B-AWQ-BF16-INT4 / Budget 12288", "effort": "budget-12288", "metrics": { "overall": 88 }, "latency": 169, "outputTokens": 11022, "reasoningTokens": 8006, "budgetTokens": 12288, "completionCap": 65536, "passed": 44, "sampleSize": 50, "completionCapHits": 2, "order": 2 }, { "id": "cyankiwi/Qwen3.6-27B-AWQ-BF16-INT4 / Free thinking", "effort": "unrestricted", "metrics": { "overall": 86 }, "latency": 289, "outputTokens": 18464, "reasoningTokens": 18080, "budgetTokens": null, "completionCap": 65536, "passed": 43, "sampleSize": 50, "completionCapHits": 7, "order": 3 } ], "note": "8,192 tokens matched unrestricted thinking at 86% with 55% less mean generation time. The 12,288 cap passed one extra task (88%); our default 1-point tolerance therefore selects it. Set the allowed gap to 2 points to select 8,192 tokens. These are vLLM token caps, not Qwen API low/medium/high modes." } ] }, { "slug": "claude-fable-5", "name": "Claude Fable 5", "provider": "Anthropic", "profileSlug": "claude-fable-5", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "Fable 5 \u00b7 Low", "effort": "low", "metrics": { "overall": 15.0 }, "cost": 1.136589, "latency": 79.822, "reportedTokens": 4707.063, "order": 2 }, { "id": "Fable 5 \u00b7 Medium", "effort": "medium", "metrics": { "overall": 14.6667 }, "cost": 1.504295, "latency": 124.995, "reportedTokens": 7237.613, "order": 3 }, { "id": "Fable 5 \u00b7 High", "effort": "high", "metrics": { "overall": 16.0 }, "cost": 1.880558, "latency": 156.708, "reportedTokens": 10213.29, "order": 4 }, { "id": "Fable 5 \u00b7 Max", "effort": "max", "metrics": { "overall": 18.0 }, "cost": 3.486334, "latency": 379.986, "reportedTokens": 27499.133, "order": 6 } ] } ] }, { "slug": "claude-fable-5-1", "name": "Claude Fable 5.1", "provider": "Anthropic", "profileSlug": "claude-fable-5-1", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "Fable 5.1 \u00b7 Low", "effort": "low", "metrics": { "overall": 16.3333 }, "cost": 0.924505, "latency": 101.654, "reportedTokens": 6069.003, "order": 2 }, { "id": "Fable 5.1 \u00b7 Medium", "effort": "medium", "metrics": { "overall": 21.0 }, "cost": 1.182898, "latency": 126.329, "reportedTokens": 8117.263, "order": 3 }, { "id": "Fable 5.1 \u00b7 High", "effort": "high", "metrics": { "overall": 20.3333 }, "cost": 1.571076, "latency": 192.618, "reportedTokens": 11420.173, "order": 4 }, { "id": "Fable 5.1 \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 21.6667 }, "cost": 2.495851, "latency": 296.722, "reportedTokens": 21082.637, "order": 5 }, { "id": "Fable 5.1 \u00b7 Max", "effort": "max", "metrics": { "overall": 27.0 }, "cost": 3.358707, "latency": 418.999, "reportedTokens": 31331.52, "order": 6 } ] } ] }, { "slug": "claude-haiku-5-5", "name": "Claude Haiku 5.5", "provider": "Anthropic", "profileSlug": "claude-haiku-5-5", "studies": [ { "id": "livebench-2026_06_25-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-06-25 \u00b7 Thinking", "scope": "Thinking", "questionSet": "2026-06-25", "dataUrl": "https://livebench.ai/table_2026_06_25.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-haiku-5-5-xhigh-effort", "effort": "xhigh", "metrics": { "Reasoning": 81.2308, "Coding": 76.393, "Agentic Coding": 51.3637, "Mathematics": 92.7525, "Data Analysis": 72.753, "Language": 63.516, "IF": 66.4665, "overall": 72.0679, "balanced": 69.8108, "coding": 64.8628, "research": 73.0774, "writing": 67.5009 }, "order": 5 }, { "id": "claude-haiku-5-5-max-effort", "effort": "max", "metrics": { "Reasoning": 84.6345, "Coding": 77.863, "Agentic Coding": 57.1717, "Mathematics": 86.435, "Data Analysis": 51.7833, "Language": 62.628, "IF": 54.6583, "overall": 67.882, "balanced": 66.043, "coding": 67.2471, "research": 65.4829, "writing": 62.3426 }, "order": 6 } ] } ] }, { "slug": "claude-opus-4-5", "name": "Claude Opus 4.5", "provider": "Anthropic", "profileSlug": "claude-opus-4-5", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 64K thinking budget", "scope": "64K thinking budget", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-opus-4-5-20251101-thinking-64k-low-effort", "effort": "low", "metrics": { "Reasoning": 70.4423, "Coding": 77.48, "Agentic Coding": 50.0, "Mathematics": 76.6527, "Data Analysis": 53.0227, "Language": 80.751, "IF": 50.8125, "overall": 65.5945, "balanced": 59.5372, "coding": 61.3916, "research": 62.8367, "writing": 65.7324 }, "order": 2 }, { "id": "claude-opus-4-5-20251101-thinking-64k-medium-effort", "effort": "medium", "metrics": { "Reasoning": 82.2067, "Coding": 77.48, "Agentic Coding": 58.3333, "Mathematics": 85.2405, "Data Analysis": 72.3487, "Language": 80.3503, "IF": 60.5168, "overall": 73.7823, "balanced": 70.0582, "coding": 67.8767, "research": 74.6329, "writing": 71.7037 }, "order": 3 }, { "id": "claude-opus-4-5-20251101-thinking-64k-high-effort", "effort": "high", "metrics": { "Reasoning": 80.0865, "Coding": 79.654, "Agentic Coding": 63.3333, "Mathematics": 90.389, "Data Analysis": 74.4417, "Language": 81.261, "IF": 62.546, "overall": 75.9588, "balanced": 71.4608, "coding": 70.6638, "research": 75.0611, "writing": 72.6631 }, "order": 4 } ] }, { "id": "livebench-2026_01_08-no-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Without extended thinking", "scope": "Without extended thinking", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-opus-4-5-20251101-low-effort", "effort": "low", "metrics": { "Reasoning": 47.8125, "Coding": 78.1845, "Agentic Coding": 50.0, "Mathematics": 64.1033, "Data Analysis": 44.1667, "Language": 77.1787, "IF": 28.9543, "overall": 55.7714, "balanced": 46.0258, "coding": 56.0226, "research": 47.352, "writing": 51.0728 }, "order": 2 }, { "id": "claude-opus-4-5-20251101-medium-effort", "effort": "medium", "metrics": { "Reasoning": 53.2115, "Coding": 78.506, "Agentic Coding": 63.3333, "Mathematics": 66.3178, "Data Analysis": 45.5413, "Language": 78.6567, "IF": 28.1085, "overall": 59.0964, "balanced": 50.339, "coding": 62.8444, "research": 49.7066, "writing": 52.0932 }, "order": 3 }, { "id": "claude-opus-4-5-20251101-high-effort", "effort": "high", "metrics": { "Reasoning": 54.9038, "Coding": 77.8015, "Agentic Coding": 61.6667, "Mathematics": 66.445, "Data Analysis": 45.6123, "Language": 77.0917, "IF": 26.5917, "overall": 58.5875, "balanced": 50.0744, "coding": 61.9852, "research": 49.7821, "writing": 51.0385 }, "order": 4 } ] } ] }, { "slug": "claude-opus-4-7", "name": "Claude Opus 4.7", "provider": "Anthropic", "profileSlug": "claude-opus-4-7", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Thinking", "scope": "Thinking", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-opus-4-7-low-effort", "effort": "low", "metrics": { "Reasoning": 74.8365, "Coding": 76.4545, "Agentic Coding": 60.0, "Mathematics": 76.3405, "Data Analysis": 75.5977, "Language": 74.5567, "IF": 46.1123, "overall": 69.1283, "balanced": 64.9641, "coding": 65.773, "research": 69.278, "writing": 61.7987 }, "order": 2 }, { "id": "claude-opus-4-7-medium-effort", "effort": "medium", "metrics": { "Reasoning": 80.0337, "Coding": 79.9755, "Agentic Coding": 56.6667, "Mathematics": 86.3792, "Data Analysis": 75.75, "Language": 76.5297, "IF": 50.8667, "overall": 72.3145, "balanced": 67.4202, "coding": 66.5844, "research": 72.3896, "writing": 65.507 }, "order": 3 }, { "id": "claude-opus-4-7-high-effort", "effort": "high", "metrics": { "Reasoning": 83.827, "Coding": 83.175, "Agentic Coding": 61.6667, "Mathematics": 89.293, "Data Analysis": 77.269, "Language": 74.4783, "IF": 54.5, "overall": 74.887, "balanced": 71.0143, "coding": 70.7266, "research": 74.5919, "writing": 66.8904 }, "order": 4 }, { "id": "claude-opus-4-7-xhigh-effort", "effort": "xhigh", "metrics": { "Reasoning": 87.6923, "Coding": 82.088, "Agentic Coding": 60.0, "Mathematics": 93.1038, "Data Analysis": 78.2637, "Language": 77.914, "IF": 59.3417, "overall": 76.9148, "balanced": 73.0915, "coding": 70.7144, "research": 77.7268, "writing": 71.0232 }, "order": 5 } ] } ] }, { "slug": "claude-opus-4-8", "name": "Claude Opus 4.8", "provider": "Anthropic", "profileSlug": "claude-opus-4-8", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "Opus 4.8 \u00b7 Low", "effort": "low", "metrics": { "overall": 12.0 }, "cost": 0.60302, "latency": 96.2, "reportedTokens": 6889.67, "order": 2 }, { "id": "Opus 4.8 \u00b7 Medium", "effort": "medium", "metrics": { "overall": 17.0 }, "cost": 0.758971, "latency": 133.357, "reportedTokens": 10074.123, "order": 3 }, { "id": "Opus 4.8 \u00b7 High", "effort": "high", "metrics": { "overall": 14.0 }, "cost": 0.857424, "latency": 186.73, "reportedTokens": 12218.97, "order": 4 }, { "id": "Opus 4.8 \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 18.6667 }, "cost": 1.200635, "latency": 251.027, "reportedTokens": 20038.797, "order": 5 }, { "id": "Opus 4.8 \u00b7 Max", "effort": "max", "metrics": { "overall": 15.0 }, "cost": 1.783553, "latency": 433.948, "reportedTokens": 34704.827, "order": 6 } ] }, { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Thinking", "scope": "Thinking", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-opus-4-8-low-effort", "effort": "low", "metrics": { "Reasoning": 83.5145, "Coding": 76.4545, "Agentic Coding": 38.3333, "Mathematics": 80.4873, "Data Analysis": 76.4037, "Language": 77.3477, "IF": 59.1168, "overall": 70.2368, "balanced": 66.6062, "coding": 58.6252, "research": 75.5767, "writing": 70.0688 }, "order": 2 }, { "id": "claude-opus-4-8-medium-effort", "effort": "medium", "metrics": { "Reasoning": 84.4905, "Coding": 78.8885, "Agentic Coding": 58.3333, "Mathematics": 81.6287, "Data Analysis": 77.2107, "Language": 80.3927, "IF": 58.125, "overall": 74.1528, "balanced": 71.0155, "coding": 68.4026, "research": 76.4188, "writing": 70.9869 }, "order": 3 }, { "id": "claude-opus-4-8-high-effort", "effort": "high", "metrics": { "Reasoning": 89.6923, "Coding": 76.393, "Agentic Coding": 60.0, "Mathematics": 82.0002, "Data Analysis": 78.0437, "Language": 81.644, "IF": 63.3625, "overall": 75.8765, "balanced": 74.0941, "coding": 69.708, "research": 79.7245, "writing": 74.6246 }, "order": 4 }, { "id": "claude-opus-4-8-xhigh-effort", "effort": "xhigh", "metrics": { "Reasoning": 89.7115, "Coding": 79.2715, "Agentic Coding": 60.0, "Mathematics": 84.3165, "Data Analysis": 78.344, "Language": 81.4173, "IF": 67.4458, "overall": 77.2152, "balanced": 75.4536, "coding": 70.9827, "research": 80.604, "writing": 76.3742 }, "order": 5 } ] }, { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "opus-4.8-low", "effort": "low", "metrics": { "overall": 73.5, "Math": 80.0, "Coding": 70.0, "Science": 62.5 }, "outputTokens": 3652.0, "domainTokens": { "Math": 6214.0, "Coding": 398.0, "Science": 2128.0 }, "sampleSize": 200, "order": 2 }, { "id": "opus-4.8-medium", "effort": "medium", "metrics": { "overall": 86.0, "Math": 88.0, "Coding": 86.7, "Science": 80.0 }, "outputTokens": 7712.0, "domainTokens": { "Math": 11207.0, "Coding": 3000.0, "Science": 6042.0 }, "sampleSize": 200, "order": 3 }, { "id": "opus-4.8-high", "effort": "high", "metrics": { "overall": 86.0, "Math": 91.0, "Coding": 81.7, "Science": 80.0 }, "outputTokens": 8870.0, "domainTokens": { "Math": 13934.0, "Coding": 1591.0, "Science": 7131.0 }, "sampleSize": 200, "order": 4 }, { "id": "opus-4.8-xhigh", "effort": "xhigh", "metrics": { "overall": 91.5, "Math": 90.0, "Coding": 93.3, "Science": 92.5 }, "outputTokens": 16876.0, "domainTokens": { "Math": 25038.0, "Coding": 4002.0, "Science": 15785.0 }, "sampleSize": 200, "order": 5 }, { "id": "opus-4.8-max", "effort": "max", "metrics": { "overall": 90.0, "Math": 84.0, "Coding": 100.0, "Science": 90.0 }, "outputTokens": 36544.0, "domainTokens": { "Math": 55252.0, "Coding": 6906.0, "Science": 34230.0 }, "sampleSize": 200, "order": 6 } ] } ] }, { "slug": "claude-opus-5", "name": "Claude Opus 5", "provider": "Anthropic", "profileSlug": "claude-opus-5", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "Opus 5 \u00b7 Low", "effort": "low", "metrics": { "overall": 15.6667 }, "cost": 0.744621, "latency": 98.161, "reportedTokens": 6922.533, "order": 2 }, { "id": "Opus 5 \u00b7 Medium", "effort": "medium", "metrics": { "overall": 17.0 }, "cost": 1.101234, "latency": 149.473, "reportedTokens": 11426.72, "order": 3 }, { "id": "Opus 5 \u00b7 High", "effort": "high", "metrics": { "overall": 18.0 }, "cost": 1.611204, "latency": 246.499, "reportedTokens": 18898.455, "order": 4 }, { "id": "Opus 5 \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 17.3333 }, "cost": 1.887681, "latency": 337.463, "reportedTokens": 24172.323, "order": 5 }, { "id": "Opus 5 \u00b7 Max", "effort": "max", "metrics": { "overall": 19.6667 }, "cost": 2.109061, "latency": 392.729, "reportedTokens": 29090.98, "order": 6 } ] }, { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "opus-5-low", "effort": "low", "metrics": { "overall": 85.0, "Math": 82.0, "Coding": 95.0, "Science": 77.5 }, "outputTokens": 1623.0, "domainTokens": { "Math": 2763.0, "Coding": 550.0, "Science": 384.0 }, "sampleSize": 200, "order": 2 }, { "id": "opus-5-medium", "effort": "medium", "metrics": { "overall": 85.0, "Math": 85.0, "Coding": 86.7, "Science": 82.5 }, "outputTokens": 4215.0, "domainTokens": { "Math": 6715.0, "Coding": 968.0, "Science": 2133.0 }, "sampleSize": 200, "order": 3 }, { "id": "opus-5-high", "effort": "high", "metrics": { "overall": 94.0, "Math": 93.0, "Coding": 100.0, "Science": 87.5 }, "outputTokens": 6745.0, "domainTokens": { "Math": 11442.0, "Coding": 968.0, "Science": 3666.0 }, "sampleSize": 200, "order": 4 }, { "id": "opus-5-xhigh", "effort": "xhigh", "metrics": { "overall": 95.0, "Math": 94.0, "Coding": 100.0, "Science": 90.0 }, "outputTokens": 9740.0, "domainTokens": { "Math": 15652.0, "Coding": 1929.0, "Science": 6679.0 }, "sampleSize": 200, "order": 5 }, { "id": "opus-5-max", "effort": "max", "metrics": { "overall": 94.5, "Math": 94.0, "Coding": 100.0, "Science": 87.5 }, "outputTokens": 13130.0, "domainTokens": { "Math": 21838.0, "Coding": 1845.0, "Science": 8289.0 }, "sampleSize": 200, "order": 6 } ] } ] }, { "slug": "claude-opus-5-5", "name": "Claude Opus 5.5", "provider": "Anthropic", "profileSlug": "claude-opus-5-5", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "Opus 5.5 \u00b7 Low", "effort": "low", "metrics": { "overall": 54.0 }, "cost": 0.61010294975, "latency": 110.751, "reportedTokens": 13010.66, "order": 2 }, { "id": "Opus 5.5 \u00b7 Medium", "effort": "medium", "metrics": { "overall": 62.5 }, "cost": 0.97343032025, "latency": 169.561, "reportedTokens": 22482.31, "order": 3 }, { "id": "Opus 5.5 \u00b7 High", "effort": "high", "metrics": { "overall": 65.0 }, "cost": 1.248579639, "latency": 220.141, "reportedTokens": 30534.055, "order": 4 }, { "id": "Opus 5.5 \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 67.0 }, "cost": 2.1610205525, "latency": 387.65, "reportedTokens": 62781.995, "order": 5 }, { "id": "Opus 5.5 \u00b7 Max", "effort": "max", "metrics": { "overall": 70.5 }, "cost": 3.5739114765, "latency": 677.163, "reportedTokens": 115101.01, "order": 6 } ] }, { "id": "livebench-2026_06_25-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-06-25 \u00b7 Thinking", "scope": "Thinking", "questionSet": "2026-06-25", "dataUrl": "https://livebench.ai/table_2026_06_25.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-opus-5-5-xhigh-effort", "effort": "xhigh", "metrics": { "Reasoning": 90.6538, "Coding": 89.253, "Agentic Coding": 65.3533, "Mathematics": 96.803, "Data Analysis": 79.803, "Language": 85.4987, "IF": 67.0498, "overall": 82.0592, "balanced": 77.925, "coding": 76.4879, "research": 81.9045, "writing": 77.9699 }, "order": 5 }, { "id": "claude-opus-5-5-max-effort", "effort": "max", "metrics": { "Reasoning": 92.1538, "Coding": 89.253, "Agentic Coding": 71.717, "Mathematics": 97.0775, "Data Analysis": 80.307, "Language": 86.27, "IF": 65.7378, "overall": 83.2166, "balanced": 79.3953, "coding": 79.4454, "research": 82.434, "writing": 77.913 }, "order": 6 } ] } ] }, { "slug": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "provider": "Anthropic", "profileSlug": "claude-sonnet-4-6", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Adaptive thinking", "scope": "Adaptive thinking", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-sonnet-4-6-thinking-auto-low-effort", "effort": "low", "metrics": { "Reasoning": 77.4423, "Coding": 74.2805, "Agentic Coding": 63.3333, "Mathematics": 80.382, "Data Analysis": 74.6113, "Language": 71.3867, "IF": 51.6332, "overall": 70.4385, "balanced": 67.4274, "coding": 67.5638, "research": 70.5228, "writing": 63.406 }, "order": 2 }, { "id": "claude-sonnet-4-6-thinking-auto-medium-effort", "effort": "medium", "metrics": { "Reasoning": 84.7692, "Coding": 79.2715, "Agentic Coding": 60.0, "Mathematics": 86.994, "Data Analysis": 77.946, "Language": 76.1027, "IF": 63.2208, "overall": 75.472, "balanced": 72.855, "coding": 69.8189, "research": 77.1126, "writing": 71.6058 }, "order": 3 }, { "id": "claude-sonnet-4-6-thinking-auto-high-effort", "effort": "high", "metrics": { "Reasoning": 86.375, "Coding": 79.9755, "Agentic Coding": 56.6667, "Mathematics": 86.5337, "Data Analysis": 76.0567, "Language": 77.6933, "IF": 63.9165, "overall": 75.3168, "balanced": 72.631, "coding": 68.8405, "research": 77.4856, "writing": 72.796 }, "order": 4 } ] } ] }, { "slug": "claude-sonnet-5", "name": "Claude Sonnet 5", "provider": "Anthropic", "profileSlug": "claude-sonnet-5", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "Sonnet 5 \u00b7 Low", "effort": "low", "metrics": { "overall": 11.6667 }, "cost": 0.263527, "latency": 75.366, "reportedTokens": 5865.98, "order": 2 }, { "id": "Sonnet 5 \u00b7 Medium", "effort": "medium", "metrics": { "overall": 13.3333 }, "cost": 0.362365, "latency": 113.951, "reportedTokens": 9081.475, "order": 3 }, { "id": "Sonnet 5 \u00b7 High", "effort": "high", "metrics": { "overall": 13.6667 }, "cost": 0.492168, "latency": 170.359, "reportedTokens": 14321.345, "order": 4 }, { "id": "Sonnet 5 \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 14.3333 }, "cost": 0.619515, "latency": 213.636, "reportedTokens": 20164.627, "order": 5 }, { "id": "Sonnet 5 \u00b7 Max", "effort": "max", "metrics": { "overall": 12.3333 }, "cost": 1.067905, "latency": 450.945, "reportedTokens": 43089.353, "order": 6 } ] } ] }, { "slug": "claude-sonnet-5-5", "name": "Claude Sonnet 5.5", "provider": "Anthropic", "profileSlug": "claude-sonnet-5-5", "studies": [ { "id": "livebench-2026_06_25-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-06-25 \u00b7 Thinking", "scope": "Thinking", "questionSet": "2026-06-25", "dataUrl": "https://livebench.ai/table_2026_06_25.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "claude-sonnet-5-5-xhigh-effort", "effort": "xhigh", "metrics": { "Reasoning": 86.8077, "Coding": 88.8705, "Agentic Coding": 39.3437, "Mathematics": 96.6947, "Data Analysis": 78.62, "Language": 83.4297, "IF": 70.5165, "overall": 77.7547, "balanced": 72.2202, "coding": 64.4386, "research": 80.5865, "writing": 78.1255 }, "order": 5 }, { "id": "claude-sonnet-5-5-max-effort", "effort": "max", "metrics": { "Reasoning": 91.6345, "Coding": 91.366, "Agentic Coding": 56.3133, "Mathematics": 96.1317, "Data Analysis": 59.4877, "Language": 77.984, "IF": 56.7878, "overall": 75.6721, "balanced": 71.0097, "coding": 72.1748, "research": 72.9735, "writing": 70.4933 }, "order": 6 } ] } ] }, { "slug": "deepseek-v4-flash", "name": "DeepSeek V4 Flash", "provider": "DeepSeek", "profileSlug": "deepseek-v4-flash", "studies": [ { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "deepseek-v4-flash-none", "effort": "none", "metrics": { "overall": 20.5, "Math": 15.0, "Coding": 15.0, "Science": 42.5 }, "outputTokens": 3387.0, "domainTokens": { "Math": 2194.0, "Coding": 656.0, "Science": 10259.0 }, "sampleSize": 200, "order": 0 }, { "id": "deepseek-v4-flash-high", "effort": "high", "metrics": { "overall": 78.0, "Math": 79.0, "Coding": 78.3, "Science": 75.0 }, "outputTokens": 40735.0, "domainTokens": { "Math": 65219.0, "Coding": 10665.0, "Science": 30752.0 }, "sampleSize": 200, "order": 4 }, { "id": "deepseek-v4-flash-max", "effort": "max", "metrics": { "overall": 83.0, "Math": 85.0, "Coding": 90.0, "Science": 67.5 }, "outputTokens": 80870.0, "domainTokens": { "Math": 109178.0, "Coding": 36951.0, "Science": 82348.0 }, "sampleSize": 200, "order": 6 } ] } ] }, { "slug": "deepseek-v4-pro", "name": "DeepSeek V4 Pro", "provider": "DeepSeek", "profileSlug": "deepseek-v4-pro", "studies": [ { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "deepseek-v4-pro-none", "effort": "none", "metrics": { "overall": 25.0, "Math": 16.0, "Coding": 26.7, "Science": 45.0 }, "outputTokens": 1460.0, "domainTokens": { "Math": 1343.0, "Coding": 2335.0, "Science": 442.0 }, "sampleSize": 200, "order": 0 }, { "id": "deepseek-v4-pro-high", "effort": "high", "metrics": { "overall": 84.0, "Math": 85.0, "Coding": 85.0, "Science": 80.0 }, "outputTokens": 43652.0, "domainTokens": { "Math": 57981.0, "Coding": 22497.0, "Science": 39561.0 }, "sampleSize": 200, "order": 4 }, { "id": "deepseek-v4-pro-max", "effort": "max", "metrics": { "overall": 84.0, "Math": 85.0, "Coding": 88.3, "Science": 75.0 }, "outputTokens": 81408.0, "domainTokens": { "Math": 98646.0, "Coding": 53951.0, "Science": 79503.0 }, "sampleSize": 200, "order": 6 } ] } ] }, { "slug": "deepseek-v4-1-flash", "name": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "profileSlug": "deepseek-v4-1-flash", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "DeepSeek V4.1 Flash \u00b7 Low", "effort": "low", "metrics": { "overall": 13.6667 }, "cost": 0.103412, "latency": 117.736, "reportedTokens": 23309.39, "order": 2 }, { "id": "DeepSeek V4.1 Flash \u00b7 High", "effort": "high", "metrics": { "overall": 14.0 }, "cost": 0.139078, "latency": 168.424, "reportedTokens": 31679.323, "order": 4 } ] } ] }, { "slug": "gemini-3-flash-preview", "name": "Gemini 3 Flash Preview", "provider": "Google", "profileSlug": null, "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Preview", "scope": "Preview", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gemini-3-flash-preview-minimal", "effort": "minimal", "metrics": { "Reasoning": 49.1683, "Coding": 78.567, "Agentic Coding": 43.3333, "Mathematics": 68.0987, "Data Analysis": 48.3057, "Language": 78.6463, "IF": 28.3208, "overall": 56.3486, "balanced": 45.5999, "coding": 53.2774, "research": 49.1617, "writing": 51.5781 }, "order": 1 }, { "id": "gemini-3-flash-preview-high", "effort": "high", "metrics": { "Reasoning": 74.548, "Coding": 73.8975, "Agentic Coding": 40.0, "Mathematics": 84.1743, "Data Analysis": 74.7727, "Language": 84.5647, "IF": 74.8625, "overall": 72.4028, "balanced": 67.6857, "coding": 58.8377, "research": 76.1808, "writing": 78.6962 }, "order": 4 } ] } ] }, { "slug": "gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", "provider": "Google", "profileSlug": null, "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 November 2025 preview", "scope": "November 2025 preview", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gemini-3-pro-preview-11-2025-low", "effort": "low", "metrics": { "Reasoning": 70.6155, "Coding": 70.6365, "Agentic Coding": 55.0, "Mathematics": 77.734, "Data Analysis": 66.9127, "Language": 79.4917, "IF": 26.8792, "overall": 63.8957, "balanced": 56.005, "coding": 59.2212, "research": 62.0888, "writing": 54.4847 }, "order": 2 }, { "id": "gemini-3-pro-preview-11-2025-high", "effort": "high", "metrics": { "Reasoning": 77.4183, "Coding": 74.602, "Agentic Coding": 55.0, "Mathematics": 81.837, "Data Analysis": 74.3897, "Language": 84.621, "IF": 65.846, "overall": 73.3877, "balanced": 69.3056, "coding": 65.3279, "research": 75.2756, "writing": 75.0918 }, "order": 4 } ] } ] }, { "slug": "gemini-3-1-pro-preview", "name": "Gemini 3.1 Pro Preview", "provider": "Google", "profileSlug": "gemini-3-1-pro-preview", "studies": [ { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "gemini-3.1-pro-preview-low", "effort": "low", "metrics": { "overall": 79.5, "Math": 73.0, "Coding": 91.7, "Science": 77.5 }, "outputTokens": 3331.0, "domainTokens": { "Math": 4208.0, "Coding": 2652.0, "Science": 2155.0 }, "sampleSize": 200, "order": 2 }, { "id": "gemini-3.1-pro-preview-medium", "effort": "medium", "metrics": { "overall": 87.5, "Math": 87.0, "Coding": 90.0, "Science": 85.0 }, "outputTokens": 7116.0, "domainTokens": { "Math": 9805.0, "Coding": 4734.0, "Science": 3910.0 }, "sampleSize": 200, "order": 3 } ] } ] }, { "slug": "gpt-5", "name": "GPT-5", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2025_05_30-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2025-05-30 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2025-05-30", "dataUrl": "https://livebench.ai/table_2025_05_30.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5-minimal", "effort": "minimal", "metrics": { "Reasoning": 54.861, "Coding": 72.5505, "Agentic Coding": 26.6667, "Mathematics": 58.9807, "Data Analysis": 64.433, "Language": 51.016, "IF": 76.8625, "overall": 57.91, "balanced": 57.9273, "coding": 49.6805, "research": 61.5562, "writing": 63.2237 }, "order": 1 }, { "id": "gpt-5-low", "effort": "low", "metrics": { "Reasoning": 90.4723, "Coding": 74.2805, "Agentic Coding": 35.0, "Mathematics": 85.3307, "Data Analysis": 69.721, "Language": 78.7347, "IF": 88.9917, "overall": 74.6473, "balanced": 74.2758, "coding": 60.5042, "research": 82.1902, "writing": 85.111 }, "order": 2 }, { "id": "gpt-5-high", "effort": "high", "metrics": { "Reasoning": 98.1667, "Coding": 77.0975, "Agentic Coding": 46.6667, "Mathematics": 92.772, "Data Analysis": 71.6345, "Language": 80.827, "IF": 88.1125, "overall": 79.3253, "balanced": 79.2664, "coding": 67.6655, "research": 85.5952, "writing": 86.7064 }, "order": 4 } ] } ] }, { "slug": "gpt-5-mini", "name": "GPT-5 mini", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5-mini-minimal", "effort": "minimal", "metrics": { "Reasoning": 32.298, "Coding": 70.698, "Agentic Coding": 21.6667, "Mathematics": 47.5305, "Data Analysis": 41.5063, "Language": 44.7887, "IF": 20.8125, "overall": 39.9001, "balanced": 32.5216, "coding": 37.8854, "research": 34.637, "writing": 32.1258 }, "order": 1 }, { "id": "gpt-5-mini-low", "effort": "low", "metrics": { "Reasoning": 45.899, "Coding": 69.5495, "Agentic Coding": 36.6667, "Mathematics": 63.2392, "Data Analysis": 44.9873, "Language": 60.4123, "IF": 50.7125, "overall": 53.0667, "balanced": 47.4842, "coding": 49.3209, "research": 48.7652, "writing": 53.8704 }, "order": 2 }, { "id": "gpt-5-mini-high", "effort": "high", "metrics": { "Reasoning": 68.322, "Coding": 68.2025, "Agentic Coding": 46.6667, "Mathematics": 82.2005, "Data Analysis": 55.195, "Language": 75.5207, "IF": 65.271, "overall": 65.9112, "balanced": 61.2472, "coding": 58.2362, "research": 64.8535, "writing": 69.8285 }, "order": 4 } ] } ] }, { "slug": "gpt-5-nano", "name": "GPT-5 nano", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5-nano-low", "effort": "low", "metrics": { "Reasoning": 27.678, "Coding": 52.725, "Agentic Coding": 10.0, "Mathematics": 48.907, "Data Analysis": 36.5663, "Language": 35.4013, "IF": 29.0957, "overall": 34.3391, "balanced": 28.3348, "coding": 27.3788, "research": 31.7866, "writing": 31.4053 }, "order": 2 }, { "id": "gpt-5-nano-high", "effort": "high", "metrics": { "Reasoning": 40.2885, "Coding": 62.3855, "Agentic Coding": 23.3333, "Mathematics": 68.4062, "Data Analysis": 43.4057, "Language": 46.8417, "IF": 55.6998, "overall": 48.623, "balanced": 43.4276, "coding": 40.8289, "research": 45.2889, "writing": 49.8448 }, "order": 4 } ] } ] }, { "slug": "gpt-5-1", "name": "GPT-5.1", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 2025-11-13 version", "scope": "2025-11-13 version", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5.1-2025-11-13-low", "effort": "low", "metrics": { "Reasoning": 59.6442, "Coding": 80.297, "Agentic Coding": 40.0, "Mathematics": 68.3783, "Data Analysis": 45.8673, "Language": 77.8517, "IF": 47.6247, "overall": 59.9519, "balanced": 52.7093, "coding": 55.7982, "research": 55.8384, "writing": 61.5184 }, "order": 2 }, { "id": "gpt-5.1-2025-11-13-medium", "effort": "medium", "metrics": { "Reasoning": 73.9808, "Coding": 75.689, "Agentic Coding": 53.3333, "Mathematics": 79.0105, "Data Analysis": 63.2237, "Language": 78.657, "IF": 60.3042, "overall": 69.1712, "balanced": 64.9894, "coding": 63.8342, "research": 68.7198, "writing": 69.6968 }, "order": 3 }, { "id": "gpt-5.1-2025-11-13-high", "effort": "high", "metrics": { "Reasoning": 78.7933, "Coding": 72.489, "Agentic Coding": 53.3333, "Mathematics": 86.8995, "Data Analysis": 69.6063, "Language": 79.2603, "IF": 63.904, "overall": 72.0408, "balanced": 67.9705, "coding": 63.9561, "research": 73.1294, "writing": 72.2799 }, "order": 4 } ] } ] }, { "slug": "gpt-5-2", "name": "GPT-5.2", "provider": "OpenAI", "profileSlug": "gpt-5-2", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 2025-12-11 version", "scope": "2025-12-11 version", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5.2-2025-12-11-low", "effort": "low", "metrics": { "Reasoning": 71.2308, "Coding": 73.8975, "Agentic Coding": 50.0, "Mathematics": 84.7142, "Data Analysis": 52.7383, "Language": 70.1807, "IF": 54.5457, "overall": 65.3296, "balanced": 60.3062, "coding": 60.8084, "research": 62.1885, "writing": 63.3025 }, "order": 2 }, { "id": "gpt-5.2-2025-12-11-medium", "effort": "medium", "metrics": { "Reasoning": 84.173, "Coding": 72.1065, "Agentic Coding": 51.6667, "Mathematics": 92.069, "Data Analysis": 70.3693, "Language": 74.9433, "IF": 57.5252, "overall": 71.8362, "balanced": 67.7326, "coding": 63.2604, "research": 73.3179, "writing": 68.4896 }, "order": 3 }, { "id": "gpt-5.2-2025-12-11-high", "effort": "high", "metrics": { "Reasoning": 83.2115, "Coding": 76.0715, "Agentic Coding": 51.6667, "Mathematics": 93.166, "Data Analysis": 78.1633, "Language": 79.809, "IF": 61.7707, "overall": 74.837, "balanced": 70.0711, "coding": 64.7302, "research": 76.8985, "writing": 72.2022 }, "order": 4 } ] } ] }, { "slug": "gpt-5-3-codex", "name": "GPT-5.3 Codex", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Coding model", "scope": "Coding model", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5.3-codex-high", "effort": "high", "metrics": { "Reasoning": 80.1538, "Coding": 78.1845, "Agentic Coding": 55.0, "Mathematics": 87.8358, "Data Analysis": 62.688, "Language": 80.0867, "IF": 65.375, "overall": 72.7605, "balanced": 68.6115, "coding": 66.7659, "research": 71.9482, "writing": 73.4765 }, "order": 4 }, { "id": "gpt-5.3-codex-xhigh", "effort": "xhigh", "metrics": { "Reasoning": 71.423, "Coding": 77.48, "Agentic Coding": 66.6667, "Mathematics": 85.6995, "Data Analysis": 49.679, "Language": 79.18, "IF": 71.3415, "overall": 71.6385, "balanced": 67.7955, "coding": 71.0916, "research": 66.047, "writing": 74.4891 }, "order": 5 } ] } ] }, { "slug": "gpt-5-4", "name": "GPT-5.4", "provider": "OpenAI", "profileSlug": "gpt-5-4", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5.4-high", "effort": "high", "metrics": { "Reasoning": 85.6538, "Coding": 78.1845, "Agentic Coding": 46.6667, "Mathematics": 89.9813, "Data Analysis": 77.0483, "Language": 83.009, "IF": 64.9543, "overall": 75.0711, "balanced": 70.6437, "coding": 63.7988, "research": 78.5355, "writing": 75.2811 }, "order": 4 }, { "id": "gpt-5.4-xhigh", "effort": "xhigh", "metrics": { "Reasoning": 88.1155, "Coding": 77.5415, "Agentic Coding": 70.0, "Mathematics": 94.148, "Data Analysis": 79.3133, "Language": 82.6337, "IF": 70.2168, "overall": 80.2812, "balanced": 77.64, "coding": 75.0015, "research": 81.0728, "writing": 77.8683 }, "order": 5 } ] }, { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "gpt-5.4", "effort": "none", "metrics": { "overall": 23.5, "Math": 17.0, "Coding": 20.0, "Science": 45.0 }, "outputTokens": 689.0, "domainTokens": { "Math": 1209.0, "Coding": 122.0, "Science": 241.0 }, "sampleSize": 200, "order": 0 }, { "id": "gpt-5.4-low", "effort": "low", "metrics": { "overall": 78.0, "Math": 73.0, "Coding": 91.7, "Science": 70.0 }, "outputTokens": 3056.0, "domainTokens": { "Math": 4646.0, "Coding": 1181.0, "Science": 1891.0 }, "sampleSize": 200, "order": 2 }, { "id": "gpt-5.4-medium", "effort": "medium", "metrics": { "overall": 82.0, "Math": 78.0, "Coding": 93.3, "Science": 75.0 }, "outputTokens": 3177.0, "domainTokens": { "Math": 3235.0, "Coding": 2338.0, "Science": 4308.0 }, "sampleSize": 200, "order": 3 }, { "id": "gpt-5.4-high", "effort": "high", "metrics": { "overall": 80.5, "Math": 74.0, "Coding": 98.3, "Science": 70.0 }, "outputTokens": 3267.0, "domainTokens": { "Math": 659.0, "Coding": 3877.0, "Science": 7698.0 }, "sampleSize": 200, "order": 4 } ] } ] }, { "slug": "gpt-5-4-mini", "name": "GPT-5.4 mini", "provider": "OpenAI", "profileSlug": "gpt-5-4-mini", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5.4-mini-low", "effort": "low", "metrics": { "Reasoning": 40.343, "Coding": 72.906, "Agentic Coding": 23.5963, "Mathematics": 64.0895, "Data Analysis": 51.88, "Language": 57.5937, "IF": 36.3687, "overall": 49.5396, "balanced": 40.987, "coding": 42.1785, "research": 45.5969, "writing": 45.4549 }, "order": 2 }, { "id": "gpt-5.4-mini-medium", "effort": "medium", "metrics": { "Reasoning": 62.045, "Coding": 71.453, "Agentic Coding": 27.0177, "Mathematics": 70.4025, "Data Analysis": 64.293, "Language": 62.3703, "IF": 50.755, "overall": 58.3338, "balanced": 53.495, "coding": 47.9761, "research": 60.5102, "writing": 57.0946 }, "order": 3 }, { "id": "gpt-5.4-mini-high", "effort": "high", "metrics": { "Reasoning": 69.6697, "Coding": 70.94, "Agentic Coding": 38.9473, "Mathematics": 74.0985, "Data Analysis": 69.267, "Language": 65.7587, "IF": 56.2858, "overall": 63.5667, "balanced": 60.2459, "coding": 54.8873, "research": 66.2855, "writing": 62.0825 }, "order": 4 }, { "id": "gpt-5.4-mini-xhigh", "effort": "xhigh", "metrics": { "Reasoning": 72.4967, "Coding": 71.624, "Agentic Coding": 47.456, "Mathematics": 78.5555, "Data Analysis": 70.945, "Language": 71.462, "IF": 60.2653, "overall": 67.5435, "balanced": 64.1107, "coding": 59.7434, "research": 69.4297, "writing": 66.5787 }, "order": 5 } ] } ] }, { "slug": "gpt-5-4-nano", "name": "GPT-5.4 nano", "provider": "OpenAI", "profileSlug": "gpt-5-4-nano", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5.4-nano-low", "effort": "low", "metrics": { "Reasoning": 42.1283, "Coding": 69.4015, "Agentic Coding": 37.2807, "Mathematics": 65.156, "Data Analysis": 43.968, "Language": 45.5587, "IF": 37.1752, "overall": 48.6669, "balanced": 42.9238, "coding": 47.6335, "research": 42.2041, "writing": 41.2716 }, "order": 2 }, { "id": "gpt-5.4-nano-medium", "effort": "medium", "metrics": { "Reasoning": 64.4903, "Coding": 70.2565, "Agentic Coding": 40.7017, "Mathematics": 83.0957, "Data Analysis": 48.716, "Language": 51.0373, "IF": 50.945, "overall": 58.4632, "balanced": 54.5567, "coding": 54.1607, "research": 55.031, "writing": 53.0137 }, "order": 3 }, { "id": "gpt-5.4-nano-high", "effort": "high", "metrics": { "Reasoning": 72.0032, "Coding": 68.376, "Agentic Coding": 49.1227, "Mathematics": 88.5935, "Data Analysis": 52.6583, "Language": 54.7507, "IF": 53.7448, "overall": 62.7499, "balanced": 59.598, "coding": 58.793, "research": 59.9602, "writing": 56.8859 }, "order": 4 }, { "id": "gpt-5.4-nano-xhigh", "effort": "xhigh", "metrics": { "Reasoning": 81.0513, "Coding": 72.137, "Agentic Coding": 49.1227, "Mathematics": 91.2703, "Data Analysis": 67.6427, "Language": 62.468, "IF": 67.2048, "overall": 70.1281, "balanced": 68.3012, "coding": 62.6245, "research": 71.4719, "writing": 67.387 }, "order": 5 } ] } ] }, { "slug": "gpt-5-5", "name": "GPT-5.5", "provider": "OpenAI", "profileSlug": "gpt-5-5", "studies": [ { "id": "livebench-2026_01_08-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-01-08 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2026-01-08", "dataUrl": "https://livebench.ai/table_2026_01_08.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-5.5-medium", "effort": "medium", "metrics": { "Reasoning": 87.274, "Coding": 78.6285, "Agentic Coding": 16.6667, "Mathematics": 69.7695, "Data Analysis": 76.987, "Language": 85.5703, "IF": 65.7375, "overall": 68.6619, "balanced": 65.3608, "coding": 50.7534, "research": 79.6251, "writing": 76.9011 }, "order": 3 }, { "id": "gpt-5.5-high", "effort": "high", "metrics": { "Reasoning": 86.2115, "Coding": 82.5325, "Agentic Coding": 30.0, "Mathematics": 96.0692, "Data Analysis": 79.4787, "Language": 87.4273, "IF": 71.9417, "overall": 76.2373, "balanced": 70.0239, "coding": 58.3857, "research": 81.5201, "writing": 80.2764 }, "order": 4 }, { "id": "gpt-5.5-xhigh", "effort": "xhigh", "metrics": { "Reasoning": 87.7115, "Coding": 82.471, "Agentic Coding": 56.6667, "Mathematics": 96.3203, "Data Analysis": 81.0803, "Language": 87.659, "IF": 73.0418, "overall": 80.7072, "balanced": 76.3164, "coding": 70.7022, "research": 82.7803, "writing": 81.0891 }, "order": 5 } ] }, { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "gpt-5.5", "effort": "none", "metrics": { "overall": 26.0, "Math": 15.0, "Coding": 38.3, "Science": 35.0 }, "outputTokens": 260.0, "domainTokens": { "Math": 431.0, "Coding": 141.0, "Science": 9.0 }, "sampleSize": 200, "order": 0 }, { "id": "gpt-5.5-low", "effort": "low", "metrics": { "overall": 75.0, "Math": 65.0, "Coding": 95.0, "Science": 70.0 }, "outputTokens": 1603.0, "domainTokens": { "Math": 2255.0, "Coding": 738.0, "Science": 1271.0 }, "sampleSize": 200, "order": 2 }, { "id": "gpt-5.5-medium", "effort": "medium", "metrics": { "overall": 86.0, "Math": 80.0, "Coding": 96.7, "Science": 85.0 }, "outputTokens": 4692.0, "domainTokens": { "Math": 6723.0, "Coding": 1739.0, "Science": 4043.0 }, "sampleSize": 200, "order": 3 }, { "id": "gpt-5.5-high", "effort": "high", "metrics": { "overall": 88.5, "Math": 82.0, "Coding": 100.0, "Science": 87.5 }, "outputTokens": 10018.0, "domainTokens": { "Math": 14062.0, "Coding": 2853.0, "Science": 10655.0 }, "sampleSize": 200, "order": 4 }, { "id": "gpt-5.5-xhigh", "effort": "xhigh", "metrics": { "overall": 90.0, "Math": 85.0, "Coding": 100.0, "Science": 87.5 }, "outputTokens": 13271.0, "domainTokens": { "Math": 19115.0, "Coding": 4501.0, "Science": 11961.0 }, "sampleSize": 200, "order": 5 } ] }, { "id": "stet-graphql-26", "source": "stet", "name": "Repository coding \u00b7 Stet exploratory study", "scope": "26 GraphQL-go-tools tasks \u00b7 Codex 0.128.0", "defaultMetric": "tests", "exploratory": true, "metrics": [ { "key": "tests", "label": "Tests passed", "description": "Share of 26 tasks whose generated patch passed tests." }, { "key": "equivalence", "label": "Human-PR equivalence", "description": "Share judged to match the intent of the original human change. AI-judged, not a human review score." }, { "key": "review", "label": "AI review passed", "description": "Share accepted by an uncalibrated AI code reviewer." }, { "key": "all", "label": "All three passed", "description": "Share passing tests, human-PR equivalence judging and AI review together." } ], "rows": [ { "id": "GPT-5.5 Codex / low", "effort": "low", "metrics": { "tests": 80.7692, "equivalence": 15.3846, "review": 11.5385, "all": 11.5385 }, "cost": 2.65, "latency": 294.6, "sampleSize": 26, "order": 2 }, { "id": "GPT-5.5 Codex / medium", "effort": "medium", "metrics": { "tests": 80.7692, "equivalence": 42.3077, "review": 19.2308, "all": 19.2308 }, "cost": 3.13, "latency": 371.8, "sampleSize": 26, "order": 3 }, { "id": "GPT-5.5 Codex / high", "effort": "high", "metrics": { "tests": 96.1538, "equivalence": 69.2308, "review": 38.4615, "all": 38.4615 }, "cost": 4.49, "latency": 572.9, "sampleSize": 26, "order": 4 }, { "id": "GPT-5.5 Codex / xhigh", "effort": "xhigh", "metrics": { "tests": 92.3077, "equivalence": 88.4615, "review": 69.2308, "all": 65.3846 }, "cost": 9.77, "latency": 732.7, "sampleSize": 26, "order": 5 } ] } ] }, { "slug": "gpt-5-6-luna", "name": "GPT-5.6 Luna", "provider": "OpenAI", "profileSlug": "gpt-5-6-luna", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GPT-5.6 Luna \u00b7 Low", "effort": "low", "metrics": { "overall": 19.0 }, "cost": 0.013141, "latency": 61.331, "reportedTokens": 2349.41, "order": 2 }, { "id": "GPT-5.6 Luna \u00b7 Medium", "effort": "medium", "metrics": { "overall": 24.0 }, "cost": 0.018455, "latency": 78.83, "reportedTokens": 3838.337, "order": 3 }, { "id": "GPT-5.6 Luna \u00b7 High", "effort": "high", "metrics": { "overall": 31.3333 }, "cost": 0.030664, "latency": 118.097, "reportedTokens": 7884.22, "order": 4 }, { "id": "GPT-5.6 Luna \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 35.3333 }, "cost": 0.039638, "latency": 145.33, "reportedTokens": 11003.187, "order": 5 } ] } ] }, { "slug": "gpt-5-6-sol", "name": "GPT-5.6 Sol", "provider": "OpenAI", "profileSlug": "gpt-5-6-sol", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GPT-5.6 Sol \u00b7 Low", "effort": "low", "metrics": { "overall": 29.6667 }, "cost": 0.369373, "latency": 76.207, "reportedTokens": 2808.507, "order": 2 }, { "id": "GPT-5.6 Sol \u00b7 Medium", "effort": "medium", "metrics": { "overall": 39.6667 }, "cost": 0.613092, "latency": 134.261, "reportedTokens": 5560.335, "order": 3 }, { "id": "GPT-5.6 Sol \u00b7 High", "effort": "high", "metrics": { "overall": 38.0 }, "cost": 0.748522, "latency": 173.33, "reportedTokens": 7967.48, "order": 4 }, { "id": "GPT-5.6 Sol \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 40.3333 }, "cost": 0.983077, "latency": 210.172, "reportedTokens": 11427.47, "order": 5 } ] }, { "id": "joon-one-prompt-effort", "source": "joon-codex", "name": "Five efforts \u00b7 one arithmetic prompt", "scope": "One fixed arithmetic prompt \u00d7 3 repeats \u00b7 Codex Pro \u00b7 concurrency 3", "questionSet": "2026-09-30 \u00b7 speed track reasoning prompt", "defaultMetric": "overall", "exploratory": true, "resourceCalculator": "generation", "metrics": [ { "key": "overall", "label": "Arithmetic runs correct", "description": "Correct answers in three repeats of one prompt: count ordered integer pairs from 1 to 60 whose product is a perfect square; expected answer 158. One miss changes the score by 33.3 points. This is not a general reasoning benchmark." } ], "rows": [ { "id": "gpt-5.6-sol / low / reasoning prompt", "effort": "low", "metrics": { "overall": 100.0 }, "cost": null, "latency": 14.255, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 710.333333, "reasoningTokens": 497.666667, "order": 2 }, { "id": "gpt-5.6-sol / medium / reasoning prompt", "effort": "medium", "metrics": { "overall": 100.0 }, "cost": null, "latency": 22.022, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1090, "reasoningTokens": 886, "order": 3 }, { "id": "gpt-5.6-sol / high / reasoning prompt", "effort": "high", "metrics": { "overall": 100.0 }, "cost": null, "latency": 22.804, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1125.666667, "reasoningTokens": 891.333333, "order": 4 }, { "id": "gpt-5.6-sol / xhigh / reasoning prompt", "effort": "xhigh", "metrics": { "overall": 100.0 }, "cost": null, "latency": 22.054, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1224, "reasoningTokens": 1074.333333, "order": 5 }, { "id": "gpt-5.6-sol / max / reasoning prompt", "effort": "max", "metrics": { "overall": 100.0 }, "cost": null, "latency": 24.516, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1498, "reasoningTokens": 1296.333333, "order": 6 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/speed/speed_calls.csv", "note": "One arithmetic question and three repeats per setting cannot establish general accuracy. The displayed output and reasoning tokens are means of raw runs; durations are medians. All five Sol6.1 settings answered this question correctly, so extra reasoning showed no accuracy gain on this example. These observations predate Ultrafast and do not compare service tiers.", "sampleUnit": "runs" } ] }, { "slug": "gpt-5-6-terra", "name": "GPT-5.6 Terra", "provider": "OpenAI", "profileSlug": "gpt-5-6-terra", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GPT-5.6 Terra \u00b7 Low", "effort": "low", "metrics": { "overall": 27.0 }, "cost": 0.143362, "latency": 64.548, "reportedTokens": 2820.823, "order": 2 }, { "id": "GPT-5.6 Terra \u00b7 Medium", "effort": "medium", "metrics": { "overall": 29.3333 }, "cost": 0.151817, "latency": 69.391, "reportedTokens": 3165.175, "order": 3 }, { "id": "GPT-5.6 Terra \u00b7 High", "effort": "high", "metrics": { "overall": 32.6667 }, "cost": 0.2037, "latency": 95.158, "reportedTokens": 5378.26, "order": 4 }, { "id": "GPT-5.6 Terra \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 39.3333 }, "cost": 0.2972, "latency": 123.851, "reportedTokens": 9486.007, "order": 5 } ] } ] }, { "slug": "gpt-6-astra", "name": "GPT-6 Astra", "provider": "OpenAI", "profileSlug": "gpt-6-astra", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GPT-6 Astra \u00b7 Low", "effort": "low", "metrics": { "overall": 55.0 }, "cost": 0.854904, "latency": 88.55, "reportedTokens": 2737.39, "order": 2 }, { "id": "GPT-6 Astra \u00b7 Medium", "effort": "medium", "metrics": { "overall": 52.6667 }, "cost": 1.236406, "latency": 128.25, "reportedTokens": 4855.013, "order": 3 }, { "id": "GPT-6 Astra \u00b7 High", "effort": "high", "metrics": { "overall": 52.0 }, "cost": 1.615272, "latency": 186.527, "reportedTokens": 7695.993, "order": 4 }, { "id": "GPT-6 Astra \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 52.3333 }, "cost": 1.750837, "latency": 224.332, "reportedTokens": 9293.433, "order": 5 }, { "id": "GPT-6 Astra \u00b7 Max", "effort": "max", "metrics": { "overall": 55.0 }, "cost": 2.114002, "latency": 313.865, "reportedTokens": 14828.06, "order": 6 } ] }, { "id": "joon-reasoning-44", "source": "joon-codex", "name": "Math and logic \u00b7 44-problem Codex study", "scope": "44 fixed problems \u00d7 2 repeats \u00b7 Codex Pro \u00b7 concurrency 3 \u00b7 medium versus high", "questionSet": "2026-09-30 \u00b7 44 published problems", "defaultMetric": "overall", "exploratory": true, "metrics": [ { "key": "overall", "label": "Answers correct", "description": "Correct answers out of 88 observed runs on 44 fixed problems, repeated twice. Repeats are not 88 independent questions; reported confidence intervals overlap for the near-ceiling models." } ], "rows": [ { "id": "gpt-6-astra / medium / 44-problem Codex test", "effort": "medium", "metrics": { "overall": 98.8636 }, "cost": null, "latency": 8.8, "sampleSize": 88, "uniqueQuestions": 44, "passed": 87, "reasoningTokens": 484.0, "outputTokens": null, "accuracyInterval": [ 93.8, 99.8 ], "order": 3 }, { "id": "gpt-6-astra / high / 44-problem Codex test", "effort": "high", "metrics": { "overall": 97.7273 }, "cost": null, "latency": 9.6, "sampleSize": 88, "uniqueQuestions": 44, "passed": 86, "reasoningTokens": 464.0, "outputTokens": null, "accuracyInterval": [ 92.1, 99.4 ], "order": 4 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/reasoning/results.csv", "note": "These are the same 44 problems at both settings. A small difference, often driven by one question, is a reason to test your tasks rather than a universal effort recommendation. Duration is the source median including network and queue time; no API cost is estimated.", "sampleUnit": "runs" }, { "id": "joon-one-prompt-effort", "source": "joon-codex", "name": "Five efforts \u00b7 one arithmetic prompt", "scope": "One fixed arithmetic prompt \u00d7 3 repeats \u00b7 Codex Pro \u00b7 concurrency 3", "questionSet": "2026-09-30 \u00b7 speed track reasoning prompt", "defaultMetric": "overall", "exploratory": true, "resourceCalculator": "generation", "metrics": [ { "key": "overall", "label": "Arithmetic runs correct", "description": "Correct answers in three repeats of one prompt: count ordered integer pairs from 1 to 60 whose product is a perfect square; expected answer 158. One miss changes the score by 33.3 points. This is not a general reasoning benchmark." } ], "rows": [ { "id": "gpt-6-astra / low / reasoning prompt", "effort": "low", "metrics": { "overall": 100.0 }, "cost": null, "latency": 18.697, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 546.333333, "reasoningTokens": 300, "order": 2 }, { "id": "gpt-6-astra / medium / reasoning prompt", "effort": "medium", "metrics": { "overall": 100.0 }, "cost": null, "latency": 18.624, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 608.333333, "reasoningTokens": 333.333333, "order": 3 }, { "id": "gpt-6-astra / high / reasoning prompt", "effort": "high", "metrics": { "overall": 100.0 }, "cost": null, "latency": 22.362, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 673, "reasoningTokens": 386, "order": 4 }, { "id": "gpt-6-astra / xhigh / reasoning prompt", "effort": "xhigh", "metrics": { "overall": 100.0 }, "cost": null, "latency": 33.108, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1021, "reasoningTokens": 773, "order": 5 }, { "id": "gpt-6-astra / max / reasoning prompt", "effort": "max", "metrics": { "overall": 100.0 }, "cost": null, "latency": 43.972, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1422.333333, "reasoningTokens": 1202, "order": 6 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/speed/speed_calls.csv", "note": "One arithmetic question and three repeats per setting cannot establish general accuracy. The displayed output and reasoning tokens are means of raw runs; durations are medians. All five Sol6.1 settings answered this question correctly, so extra reasoning showed no accuracy gain on this example. These observations predate Ultrafast and do not compare service tiers.", "sampleUnit": "runs" } ] }, { "slug": "gpt-6-luna", "name": "GPT-6 Luna", "provider": "OpenAI", "profileSlug": "gpt-6-luna", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GPT-6 Luna \u00b7 Low", "effort": "low", "metrics": { "overall": 30.3333 }, "cost": 0.0107957625, "latency": 63.516, "reportedTokens": 2619.027, "order": 2 }, { "id": "GPT-6 Luna \u00b7 Medium", "effort": "medium", "metrics": { "overall": 42.6667 }, "cost": 0.019124674467, "latency": 116.566, "reportedTokens": 9117.587, "order": 3 }, { "id": "GPT-6 Luna \u00b7 High", "effort": "high", "metrics": { "overall": 46.0 }, "cost": 0.025823215225, "latency": 175.446, "reportedTokens": 15545.325, "order": 4 }, { "id": "GPT-6 Luna \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 46.0 }, "cost": 0.03956597605, "latency": 188.733, "reportedTokens": 19321.703, "order": 5 }, { "id": "GPT-6 Luna \u00b7 Max", "effort": "max", "metrics": { "overall": 52.3333 }, "cost": 0.057961059717, "latency": 353.172, "reportedTokens": 37928.297, "order": 6 } ] }, { "id": "joon-reasoning-44", "source": "joon-codex", "name": "Math and logic \u00b7 44-problem Codex study", "scope": "44 fixed problems \u00d7 2 repeats \u00b7 Codex Pro \u00b7 concurrency 3 \u00b7 medium versus high", "questionSet": "2026-09-30 \u00b7 44 published problems", "defaultMetric": "overall", "exploratory": true, "metrics": [ { "key": "overall", "label": "Answers correct", "description": "Correct answers out of 88 observed runs on 44 fixed problems, repeated twice. Repeats are not 88 independent questions; reported confidence intervals overlap for the near-ceiling models." } ], "rows": [ { "id": "gpt-6-luna / medium / 44-problem Codex test", "effort": "medium", "metrics": { "overall": 81.8182 }, "cost": null, "latency": 9.1, "sampleSize": 88, "uniqueQuestions": 44, "passed": 72, "reasoningTokens": 363.0, "outputTokens": null, "accuracyInterval": [ 72.5, 88.5 ], "order": 3 }, { "id": "gpt-6-luna / high / 44-problem Codex test", "effort": "high", "metrics": { "overall": 86.3636 }, "cost": null, "latency": 9.0, "sampleSize": 88, "uniqueQuestions": 44, "passed": 76, "reasoningTokens": 561.0, "outputTokens": null, "accuracyInterval": [ 77.7, 92.0 ], "order": 4 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/reasoning/results.csv", "note": "These are the same 44 problems at both settings. A small difference, often driven by one question, is a reason to test your tasks rather than a universal effort recommendation. Duration is the source median including network and queue time; no API cost is estimated.", "sampleUnit": "runs" }, { "id": "joon-one-prompt-effort", "source": "joon-codex", "name": "Five efforts \u00b7 one arithmetic prompt", "scope": "One fixed arithmetic prompt \u00d7 3 repeats \u00b7 Codex Pro \u00b7 concurrency 3", "questionSet": "2026-09-30 \u00b7 speed track reasoning prompt", "defaultMetric": "overall", "exploratory": true, "resourceCalculator": "generation", "metrics": [ { "key": "overall", "label": "Arithmetic runs correct", "description": "Correct answers in three repeats of one prompt: count ordered integer pairs from 1 to 60 whose product is a perfect square; expected answer 158. One miss changes the score by 33.3 points. This is not a general reasoning benchmark." } ], "rows": [ { "id": "gpt-6-luna / low / reasoning prompt", "effort": "low", "metrics": { "overall": 66.6667 }, "cost": null, "latency": 19.919, "sampleSize": 3, "uniqueQuestions": 1, "passed": 2, "outputTokens": 862.666667, "reasoningTokens": 663, "order": 2 }, { "id": "gpt-6-luna / medium / reasoning prompt", "effort": "medium", "metrics": { "overall": 66.6667 }, "cost": null, "latency": 16.361, "sampleSize": 3, "uniqueQuestions": 1, "passed": 2, "outputTokens": 834.666667, "reasoningTokens": 616.333333, "order": 3 }, { "id": "gpt-6-luna / high / reasoning prompt", "effort": "high", "metrics": { "overall": 100.0 }, "cost": null, "latency": 18.101, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 783.333333, "reasoningTokens": 571.333333, "order": 4 }, { "id": "gpt-6-luna / xhigh / reasoning prompt", "effort": "xhigh", "metrics": { "overall": 66.6667 }, "cost": null, "latency": 24.821, "sampleSize": 3, "uniqueQuestions": 1, "passed": 2, "outputTokens": 1415, "reasoningTokens": 1178, "order": 5 }, { "id": "gpt-6-luna / max / reasoning prompt", "effort": "max", "metrics": { "overall": 66.6667 }, "cost": null, "latency": 41.019, "sampleSize": 3, "uniqueQuestions": 1, "passed": 2, "outputTokens": 1840.666667, "reasoningTokens": 1600, "order": 6 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/speed/speed_calls.csv", "note": "One arithmetic question and three repeats per setting cannot establish general accuracy. The displayed output and reasoning tokens are means of raw runs; durations are medians. All five Sol6.1 settings answered this question correctly, so extra reasoning showed no accuracy gain on this example. These observations predate Ultrafast and do not compare service tiers.", "sampleUnit": "runs" } ] }, { "slug": "gpt-6-sol", "name": "GPT-6 Sol", "provider": "OpenAI", "profileSlug": "gpt-6-sol", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GPT-6 Sol \u00b7 Low", "effort": "low", "metrics": { "overall": 50.3333 }, "cost": 0.174839599833, "latency": 97.585, "reportedTokens": 2752.157, "order": 2 }, { "id": "GPT-6 Sol \u00b7 Medium", "effort": "medium", "metrics": { "overall": 51.3333 }, "cost": 0.270769650167, "latency": 146.184, "reportedTokens": 4774.527, "order": 3 }, { "id": "GPT-6 Sol \u00b7 High", "effort": "high", "metrics": { "overall": 54.6667 }, "cost": 0.387394600167, "latency": 238.047, "reportedTokens": 8464.567, "order": 4 }, { "id": "GPT-6 Sol \u00b7 XHigh", "effort": "xhigh", "metrics": { "overall": 61.3333 }, "cost": 0.581652172833, "latency": 326.66, "reportedTokens": 14595.657, "order": 5 }, { "id": "GPT-6 Sol \u00b7 Max", "effort": "max", "metrics": { "overall": 57.0 }, "cost": 0.984485202833, "latency": 617.514, "reportedTokens": 29926.53, "order": 6 } ] }, { "id": "joon-reasoning-44", "source": "joon-codex", "name": "Math and logic \u00b7 44-problem Codex study", "scope": "44 fixed problems \u00d7 2 repeats \u00b7 Codex Pro \u00b7 concurrency 3 \u00b7 medium versus high", "questionSet": "2026-09-30 \u00b7 44 published problems", "defaultMetric": "overall", "exploratory": true, "metrics": [ { "key": "overall", "label": "Answers correct", "description": "Correct answers out of 88 observed runs on 44 fixed problems, repeated twice. Repeats are not 88 independent questions; reported confidence intervals overlap for the near-ceiling models." } ], "rows": [ { "id": "gpt-6-sol / medium / 44-problem Codex test", "effort": "medium", "metrics": { "overall": 96.5909 }, "cost": null, "latency": 7.9, "sampleSize": 88, "uniqueQuestions": 44, "passed": 85, "reasoningTokens": 316.0, "outputTokens": null, "accuracyInterval": [ 90.5, 98.8 ], "order": 3 }, { "id": "gpt-6-sol / high / 44-problem Codex test", "effort": "high", "metrics": { "overall": 97.7273 }, "cost": null, "latency": 8.1, "sampleSize": 88, "uniqueQuestions": 44, "passed": 86, "reasoningTokens": 427.0, "outputTokens": null, "accuracyInterval": [ 92.1, 99.4 ], "order": 4 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/reasoning/results.csv", "note": "These are the same 44 problems at both settings. A small difference, often driven by one question, is a reason to test your tasks rather than a universal effort recommendation. Duration is the source median including network and queue time; no API cost is estimated.", "sampleUnit": "runs" }, { "id": "joon-one-prompt-effort", "source": "joon-codex", "name": "Five efforts \u00b7 one arithmetic prompt", "scope": "One fixed arithmetic prompt \u00d7 3 repeats \u00b7 Codex Pro \u00b7 concurrency 3", "questionSet": "2026-09-30 \u00b7 speed track reasoning prompt", "defaultMetric": "overall", "exploratory": true, "resourceCalculator": "generation", "metrics": [ { "key": "overall", "label": "Arithmetic runs correct", "description": "Correct answers in three repeats of one prompt: count ordered integer pairs from 1 to 60 whose product is a perfect square; expected answer 158. One miss changes the score by 33.3 points. This is not a general reasoning benchmark." } ], "rows": [ { "id": "gpt-6-sol / low / reasoning prompt", "effort": "low", "metrics": { "overall": 66.6667 }, "cost": null, "latency": 9.438, "sampleSize": 3, "uniqueQuestions": 1, "passed": 2, "outputTokens": 384.666667, "reasoningTokens": 250.666667, "order": 2 }, { "id": "gpt-6-sol / medium / reasoning prompt", "effort": "medium", "metrics": { "overall": 66.6667 }, "cost": null, "latency": 14.284, "sampleSize": 3, "uniqueQuestions": 1, "passed": 2, "outputTokens": 590.333333, "reasoningTokens": 441.666667, "order": 3 }, { "id": "gpt-6-sol / high / reasoning prompt", "effort": "high", "metrics": { "overall": 100.0 }, "cost": null, "latency": 13.253, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 652, "reasoningTokens": 492.333333, "order": 4 }, { "id": "gpt-6-sol / xhigh / reasoning prompt", "effort": "xhigh", "metrics": { "overall": 66.6667 }, "cost": null, "latency": 14.361, "sampleSize": 3, "uniqueQuestions": 1, "passed": 2, "outputTokens": 758.666667, "reasoningTokens": 616, "order": 5 }, { "id": "gpt-6-sol / max / reasoning prompt", "effort": "max", "metrics": { "overall": 100.0 }, "cost": null, "latency": 22.587, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1184, "reasoningTokens": 1025.333333, "order": 6 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/speed/speed_calls.csv", "note": "One arithmetic question and three repeats per setting cannot establish general accuracy. The displayed output and reasoning tokens are means of raw runs; durations are medians. All five Sol6.1 settings answered this question correctly, so extra reasoning showed no accuracy gain on this example. These observations predate Ultrafast and do not compare service tiers.", "sampleUnit": "runs" } ] }, { "slug": "gpt-6-1-sol", "name": "GPT-6.1 Sol", "provider": "OpenAI", "profileSlug": "gpt-6-1-sol", "studies": [ { "id": "livebench-2026_06_25-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2026-06-25 \u00b7 Explicit effort rows", "scope": "Explicit effort rows", "questionSet": "2026-06-25", "dataUrl": "https://livebench.ai/table_2026_06_25.csv", "defaultMetric": "balanced", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 7 LiveBench category scores. Each effort uses the same question set." }, { "key": "balanced", "label": "General agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "coding", "label": "Coding agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Agentic Coding", "label": "Agentic Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "gpt-6.1-sol-xhigh", "effort": "xhigh", "metrics": { "Reasoning": 91.6345, "Coding": 80.741, "Agentic Coding": 56.7677, "Mathematics": 96.4785, "Data Analysis": 82.1797, "Language": 88.645, "IF": 71.325, "overall": 81.1102, "balanced": 77.0762, "coding": 70.6454, "research": 84.2877, "writing": 81.2994 }, "order": 5 }, { "id": "gpt-6.1-sol-max", "effort": "max", "metrics": { "Reasoning": 92.6345, "Coding": 80.3585, "Agentic Coding": 54.5453, "Mathematics": 96.8317, "Data Analysis": 82.6593, "Language": 90.126, "IF": 74.1542, "overall": 81.6157, "balanced": 77.6727, "coding": 69.9635, "research": 85.5696, "writing": 83.315 }, "order": 6 } ] }, { "id": "joon-reasoning-44", "source": "joon-codex", "name": "Math and logic \u00b7 44-problem Codex study", "scope": "44 fixed problems \u00d7 2 repeats \u00b7 Codex Pro \u00b7 concurrency 3 \u00b7 medium versus high", "questionSet": "2026-09-30 \u00b7 44 published problems", "defaultMetric": "overall", "exploratory": true, "metrics": [ { "key": "overall", "label": "Answers correct", "description": "Correct answers out of 88 observed runs on 44 fixed problems, repeated twice. Repeats are not 88 independent questions; reported confidence intervals overlap for the near-ceiling models." } ], "rows": [ { "id": "gpt-6.1-sol / medium / 44-problem Codex test", "effort": "medium", "metrics": { "overall": 97.7273 }, "cost": null, "latency": 10.1, "sampleSize": 88, "uniqueQuestions": 44, "passed": 86, "reasoningTokens": 591.0, "outputTokens": null, "accuracyInterval": [ 92.1, 99.4 ], "order": 3 }, { "id": "gpt-6.1-sol / high / 44-problem Codex test", "effort": "high", "metrics": { "overall": 100.0 }, "cost": null, "latency": 11.0, "sampleSize": 88, "uniqueQuestions": 44, "passed": 88, "reasoningTokens": 609.0, "outputTokens": null, "accuracyInterval": [ 95.8, 100.0 ], "order": 4 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/reasoning/results.csv", "note": "These are the same 44 problems at both settings. A small difference, often driven by one question, is a reason to test your tasks rather than a universal effort recommendation. Duration is the source median including network and queue time; no API cost is estimated.", "sampleUnit": "runs" }, { "id": "joon-one-prompt-effort", "source": "joon-codex", "name": "Five efforts \u00b7 one arithmetic prompt", "scope": "One fixed arithmetic prompt \u00d7 3 repeats \u00b7 Codex Pro \u00b7 concurrency 3", "questionSet": "2026-09-30 \u00b7 speed track reasoning prompt", "defaultMetric": "overall", "exploratory": true, "resourceCalculator": "generation", "metrics": [ { "key": "overall", "label": "Arithmetic runs correct", "description": "Correct answers in three repeats of one prompt: count ordered integer pairs from 1 to 60 whose product is a perfect square; expected answer 158. One miss changes the score by 33.3 points. This is not a general reasoning benchmark." } ], "rows": [ { "id": "gpt-6.1-sol / low / reasoning prompt", "effort": "low", "metrics": { "overall": 100.0 }, "cost": null, "latency": 17.626, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 494.666667, "reasoningTokens": 271.333333, "order": 2 }, { "id": "gpt-6.1-sol / medium / reasoning prompt", "effort": "medium", "metrics": { "overall": 100.0 }, "cost": null, "latency": 19.76, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 593, "reasoningTokens": 342.666667, "order": 3 }, { "id": "gpt-6.1-sol / high / reasoning prompt", "effort": "high", "metrics": { "overall": 100.0 }, "cost": null, "latency": 26.837, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 756.333333, "reasoningTokens": 487.333333, "order": 4 }, { "id": "gpt-6.1-sol / xhigh / reasoning prompt", "effort": "xhigh", "metrics": { "overall": 100.0 }, "cost": null, "latency": 43.683, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1334, "reasoningTokens": 1086.666667, "order": 5 }, { "id": "gpt-6.1-sol / max / reasoning prompt", "effort": "max", "metrics": { "overall": 100.0 }, "cost": null, "latency": 53.478, "sampleSize": 3, "uniqueQuestions": 1, "passed": 3, "outputTokens": 1543.666667, "reasoningTokens": 1328.333333, "order": 6 } ], "dataUrl": "https://github.com/joonlab/gpt-6.1-sol-benchmark/blob/b38279fd8e28e4534899000d88427af0ddec640b/data/speed/speed_calls.csv", "note": "One arithmetic question and three repeats per setting cannot establish general accuracy. The displayed output and reasoning tokens are means of raw runs; durations are medians. All five Sol6.1 settings answered this question correctly, so extra reasoning showed no accuracy gain on this example. These observations predate Ultrafast and do not compare service tiers.", "sampleUnit": "runs" } ], "externalReferences": [ { "name": "Artificial Analysis \u00b7 GPT-6.1 Sol effort results", "url": "https://artificialanalysis.ai/models/gpt-6-1-sol", "checked": "2026-10-03", "note": "The publisher now lists low, medium, high, extra high and max. Read its current comparison and methodology. Numeric rows are not imported here because reuse permission has not been verified." }, { "name": "Hex DataBench \u00b7 GPT-6.1 Sol five-level analytics study", "url": "https://hex.tech/databench/", "checked": "2026-10-07", "note": "The publisher now lists five tested effort levels. Its current table is linked while reuse permission is unresolved; new numeric rows are not included here. Our existing matched LiveBench study uses xhigh and max." } ] }, { "slug": "o3", "name": "o3", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2025_04_25-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2025-04-25 \u00b7 2025-04-16 version", "scope": "2025-04-16 version", "questionSet": "2025-04-25", "dataUrl": "https://livebench.ai/table_2025_04_25.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 6 LiveBench category scores. Each effort uses the same question set." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "o3-2025-04-16-medium", "effort": "medium", "metrics": { "Reasoning": 91.0, "Coding": 77.863, "Mathematics": 80.6573, "Data Analysis": 68.1925, "Language": 73.4813, "IF": 84.3208, "overall": 79.2525, "research": 80.1941, "writing": 80.9869 }, "order": 3 }, { "id": "o3-2025-04-16-high", "effort": "high", "metrics": { "Reasoning": 93.3333, "Coding": 76.7145, "Mathematics": 85.0037, "Data Analysis": 67.02, "Language": 75.996, "IF": 86.1748, "overall": 80.707, "research": 81.407, "writing": 83.177 }, "order": 4 } ] } ] }, { "slug": "o3-mini", "name": "o3-mini", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2025_04_02-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2025-04-02 \u00b7 2025-01-31 version", "scope": "2025-01-31 version", "questionSet": "2025-04-02", "dataUrl": "https://livebench.ai/table_2025_04_02.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 6 LiveBench category scores. Each effort uses the same question set." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "o3-mini-2025-01-31-low", "effort": "low", "metrics": { "Reasoning": 55.9443, "Coding": 50.762, "Mathematics": 61.6747, "Data Analysis": 62.04, "Language": 48.0707, "IF": 80.0625, "overall": 59.759, "research": 61.4156, "writing": 63.648 }, "order": 2 }, { "id": "o3-mini-2025-01-31-medium", "effort": "medium", "metrics": { "Reasoning": 69.0, "Coding": 58.4285, "Mathematics": 71.6753, "Data Analysis": 66.56, "Language": 54.1173, "IF": 83.1582, "overall": 67.1566, "research": 68.8672, "writing": 69.4181 }, "order": 3 }, { "id": "o3-mini-2025-01-31-high", "effort": "high", "metrics": { "Reasoning": 74.361, "Coding": 65.4765, "Mathematics": 76.5543, "Data Analysis": 70.64, "Language": 56.8553, "IF": 84.3585, "overall": 71.3743, "research": 72.6183, "writing": 71.8576 }, "order": 4 } ] } ] }, { "slug": "o4-mini", "name": "o4-mini", "provider": "OpenAI", "profileSlug": null, "studies": [ { "id": "livebench-2025_04_25-thinking", "source": "livebench", "name": "General tasks \u00b7 LiveBench 2025-04-25 \u00b7 2025-04-16 version", "scope": "2025-04-16 version", "questionSet": "2025-04-25", "dataUrl": "https://livebench.ai/table_2025_04_25.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall benchmark", "description": "Equal-weight average of the 6 LiveBench category scores. Each effort uses the same question set." }, { "key": "research", "label": "Research agent fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "writing", "label": "Writing & support fit", "description": "AI Agent Store task-weighted category mix. An editorial selection aid, not a measured agent success rate; weights match our published Agent Fit methodology." }, { "key": "Reasoning", "label": "Reasoning", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Coding", "label": "Coding", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Mathematics", "label": "Mathematics", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Data Analysis", "label": "Data Analysis", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "Language", "label": "Language", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." }, { "key": "IF", "label": "Instruction following", "description": "Published LiveBench task scores averaged within this category; same question set for all effort levels in this sweep." } ], "rows": [ { "id": "o4-mini-2025-04-16-medium", "effort": "medium", "metrics": { "Reasoning": 78.4723, "Coding": 74.219, "Mathematics": 81.02, "Data Analysis": 68.4715, "Language": 62.409, "IF": 81.825, "overall": 74.4028, "research": 73.7331, "writing": 73.5557 }, "order": 3 }, { "id": "o4-mini-2025-04-16-high", "effort": "high", "metrics": { "Reasoning": 88.111, "Coding": 79.9755, "Mathematics": 84.8953, "Data Analysis": 68.3275, "Language": 66.0547, "IF": 84.9582, "overall": 78.7204, "research": 78.2369, "writing": 77.8697 }, "order": 4 } ] } ] }, { "slug": "glm-5-2", "name": "GLM 5.2", "provider": "Z.ai", "profileSlug": "glm-5-2", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GLM 5.2 \u00b7 High", "effort": "high", "metrics": { "overall": 13.3333 }, "cost": 0.253753, "latency": 136.744, "reportedTokens": 10494.41, "order": 4 }, { "id": "GLM 5.2 \u00b7 Max", "effort": "max", "metrics": { "overall": 9.0 }, "cost": 0.320113, "latency": 183.974, "reportedTokens": 19073.47, "order": 6 } ] }, { "id": "ockbench-200", "source": "ockbench", "name": "Math, code & science \u00b7 OckBench", "scope": "200 selected problems", "dataUrl": "https://ockbench.github.io/static/data/top200_model_performance.csv", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Overall accuracy", "description": "Correct answers out of 200 problems. Domain proportions are 100 math, 60 coding and 40 science." }, { "key": "Math", "label": "Mathematics", "description": "Accuracy on 100 selected mathematics problems, graded with an LLM judge." }, { "key": "Coding", "label": "Coding tests", "description": "Accuracy on 60 selected programming problems checked with executable tests." }, { "key": "Science", "label": "Science", "description": "Accuracy on 40 selected science problems with checked multiple-choice answers." } ], "rows": [ { "id": "glm-5.2-off", "effort": "off", "metrics": { "overall": 28.5, "Math": 13.0, "Coding": 40.0, "Science": 50.0 }, "outputTokens": 2090.0, "domainTokens": { "Math": 3009.0, "Coding": 1014.0, "Science": 1404.0 }, "sampleSize": 200, "order": 0 }, { "id": "glm-5.2-high", "effort": "high", "metrics": { "overall": 78.5, "Math": 76.0, "Coding": 85.0, "Science": 75.0 }, "outputTokens": 38251.0, "domainTokens": { "Math": 40941.0, "Coding": 27061.0, "Science": 47749.0 }, "sampleSize": 200, "order": 4 }, { "id": "glm-5.2-max", "effort": "max", "metrics": { "overall": 83.5, "Math": 79.0, "Coding": 88.3, "Science": 87.5 }, "outputTokens": 41234.0, "domainTokens": { "Math": 42965.0, "Coding": 34340.0, "Science": 46900.0 }, "sampleSize": 200, "order": 6 } ] } ] }, { "slug": "glm-5-3", "name": "GLM 5.3", "provider": "Z.ai", "profileSlug": "glm-5-3", "studies": [ { "id": "databench-v1-1", "source": "databench", "name": "Analytics \u00b7 DataBench v1.1", "scope": "Analytics in the Hex agent harness", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Analytics score", "description": "Published blended DataBench score, on a 0\u2013100 scale. Judge-based analytical task performance; not a general intelligence or guaranteed success rate." } ], "rows": [ { "id": "GLM 5.3 \u00b7 Low", "effort": "low", "metrics": { "overall": 8.6667 }, "cost": 0.199005, "latency": 94.651, "reportedTokens": 5341.487, "order": 2 }, { "id": "GLM 5.3 \u00b7 High", "effort": "high", "metrics": { "overall": 14.6667 }, "cost": 0.356431, "latency": 215.078, "reportedTokens": 14874.81, "order": 4 }, { "id": "GLM 5.3 \u00b7 Max", "effort": "max", "metrics": { "overall": 16.3333 }, "cost": 0.638527, "latency": 518.487, "reportedTokens": 38848.457, "order": 6 } ] } ] }, { "slug": "grok-4-6", "name": "Grok 4.6", "provider": "xAI", "profileSlug": "grok-4-6", "studies": [ { "id": "aa-intelligence-v4-3-2", "source": "artificial-analysis", "name": "General capability \u00b7 Artificial Analysis v4.3.2", "scope": "Grok 4.6 API \u00b7 Intelligence Index v4.3.2 \u00b7 four effort settings", "defaultMetric": "overall", "metrics": [ { "key": "overall", "label": "Intelligence Index", "description": "Artificial Analysis composite of ten evaluations, on a 0\u2013100 scale. Index points are not percent correct. The same index version is used for every effort; costs and tokens reflect the publisher\u2019s weighted task mix." } ], "rows": [ { "id": "grok-4-6-low", "effort": "low", "metrics": { "overall": 35.1246953482174 }, "cost": 0.47528534374531, "outputTokens": 10264.013906326, "reasoningTokens": 2460.8077250036104, "sourceUrl": "https://artificialanalysis.ai/models/grok-4-6-low", "order": 2 }, { "id": "grok-4-6-medium", "effort": "medium", "metrics": { "overall": 42.836322654063 }, "cost": 1.4963233956114492, "outputTokens": 28466.40897482784, "reasoningTokens": 13758.04273213306, "sourceUrl": "https://artificialanalysis.ai/models/grok-4-6-medium", "order": 3 }, { "id": "grok-4-6", "effort": "high", "metrics": { "overall": 44.3113073012592 }, "cost": 1.8589423352127492, "outputTokens": 35834.50984907935, "reasoningTokens": 18781.562301591235, "sourceUrl": "https://artificialanalysis.ai/models/grok-4-6", "order": 4 }, { "id": "grok-4-6-xhigh", "effort": "xhigh", "metrics": { "overall": 44.199810436996 }, "cost": 2.323692124283027, "outputTokens": 37608.15138851136, "reasoningTokens": 19801.67585008191, "sourceUrl": "https://artificialanalysis.ai/models/grok-4-6-xhigh", "order": 5 } ], "note": "High is the default near-best pick. Extra high uses more money for a slightly lower observed index score. The small gap is not evidence of a reliable quality difference. Increase the allowed gap to 1.5 points to include medium." } ] } ], "sourceHashes": { "reports/model-effort-research/categories_2024_06_24.json": "08fa233f92ca970a786f4ab8ea09639bce4c9dcbc88565254989e0bd34d08bfa", "reports/model-effort-research/categories_2024_07_26.json": "31382962c060909421a3f2d8fe9f2194daadc252f654647daa8f684efbbe674b", "reports/model-effort-research/categories_2024_08_31.json": "31382962c060909421a3f2d8fe9f2194daadc252f654647daa8f684efbbe674b", "reports/model-effort-research/categories_2024_11_25.json": "31382962c060909421a3f2d8fe9f2194daadc252f654647daa8f684efbbe674b", "reports/model-effort-research/categories_2025_04_02.json": "182ba3b11478a1f1768020fc79b6817b5698054333d768a8c6c800f65cd90cfa", "reports/model-effort-research/categories_2025_04_25.json": "742ed9e6946db311c91361781d71d3f6c3587b5ade5daf7373eb32a39d21f5fb", "reports/model-effort-research/categories_2025_05_30.json": "f79b260f8c39b9763007dda694c2ee3f11b9898e17a85ee1c353c8a0b5a8de22", "reports/model-effort-research/categories_2025_11_25.json": "4b407579b95ffacb519a33afc07affc78096af029ffd29891bd3bcd18b62994f", "reports/model-effort-research/categories_2025_12_23.json": "e8137ae3bf1baabec1ae22d4815cb3f8cb5ed4abf8bae63c2261f3fd06fa964a", "reports/model-effort-research/categories_2026_01_08.json": "dad300ad18655b69db720e1b88fc5a5eac06c5b2f0e52c2bf50f10ff057674f3", "reports/model-effort-research/categories_2026_06_25.json": "dad300ad18655b69db720e1b88fc5a5eac06c5b2f0e52c2bf50f10ff057674f3", "reports/model-effort-research/cost_2026_06_25.csv": "66bb9fb719d29ebd296da6f04d90a721d33c4aaf588e5ae223eb3c88b299bd4f", "reports/model-effort-research/databench-rows.json": "1d2e9b1467a56c8b6b8545bbeb3d8a2a5f0bde7952ec9385b8037328270016b1", "reports/model-effort-research/grok46-aa-reviewed.json": "968ee117f9f0a14836f0f882c2b3483456cc86e245857815407631fb0b07e5c4", "reports/model-effort-research/livebench-files.json": "e98289a852da3f971125c69d5b08ed0745e8863e0b2ab2bf5313f824a97f3dbc", "reports/model-effort-research/livebench-models.js": "f96c19bf94e15e6ad2cd03b59e4d7efe176292834820058b44c58c0c1098dfef", "reports/model-effort-research/livebench-releases.js": "ce56014792362d8c36d3e38d00d61709b849d7c572080c58eef53a4aa748f14d", "reports/model-effort-research/ock.js": "b86813ac84c3eb4fc702867a773d3a9f7d6d7be3d972619c018890f521f00856", "reports/model-effort-research/per_domain_performance.csv": "796f18559ad0e7669879ac0115f4950fce9e0e71a758111a13749d7be739b808", "reports/model-effort-research/qwen36-budget-reviewed.json": "a41ecc8af0d39db0ee05a9edf149f432f15bd43f6ad6bab91f84eda0c3b2d442", "reports/model-effort-research/table_2024_06_24.csv": "df181166a5a670a5b7678ee02f96c6896d8a88a503aec20243234b698d25a8c0", "reports/model-effort-research/table_2024_07_26.csv": "670dc4af2497d2daca7b5179f9558854163d954a95939bc4a7d71d8fce298a85", "reports/model-effort-research/table_2024_08_31.csv": "8c66d248110076378f7eb58aaeef524c65bba6dff0b55f190fd9ac8e92d73b2a", "reports/model-effort-research/table_2024_11_25.csv": "17ca6b3d122921d24b591d0da52566c5552ed4f4001ae9d83a11b2712325aae4", "reports/model-effort-research/table_2025_04_02.csv": "d7fd6a3e461cff4f96569093553789115e87adeb9cd95dca043bafa51ff2acaa", "reports/model-effort-research/table_2025_04_25.csv": "251eb7461da69232733b80d1e63f00a9db5b921d90c966d139ce0e2851c762d9", "reports/model-effort-research/table_2025_05_30.csv": "c1215f3df5b2e0a5fe69907c101d29922bf783b011df9ba82638c0145cb4c64c", "reports/model-effort-research/table_2025_11_25.csv": "34242000938912ac33369e3b25d49f6eafd3cc64823914b2405cef9a7ddcd81d", "reports/model-effort-research/table_2025_12_23.csv": "d8a96fd89213f11f144c92846567f8652bc2740511e8b4bc4211f659ff57f670", "reports/model-effort-research/table_2026_01_08.csv": "bcb52d78b463adeccdd06899452a0bf68ffa84906bd37f68fdc0659d9f241241", "reports/model-effort-research/table_2026_06_25.csv": "05f59189a82813c446681b0c5902a559ef9c5d55263a2bb0ae2d1868fb5ecb2d", "reports/model-effort-research/top200_model_performance.csv": "7c685e6418ab46046cdd1307801b946cd586750d831c501da78b09f95700d718", "reports/model-refresh/2026-10-15/table_2026_06_25.csv": "bf9ce6130134893d61157c214865de13d265a5e30790ff6cd4e3142a25113799", "reports/model-refresh/2026-10-15/categories_2026_06_25.json": "dad300ad18655b69db720e1b88fc5a5eac06c5b2f0e52c2bf50f10ff057674f3", "reports/model-refresh/2026-10-15/review.json": "9c546583447e58c72eeb9ccd42ee7c723655e625dd5c4982dbdd1f07b98d54fd", "reports/model-refresh/2026-10-15/joon-data-reasoning-problems.csv": "03e924b42d37c64c6cc0760864196b4d33e48d437a16e34db56115a2dd083584", "reports/model-refresh/2026-10-15/joon-data-speed-speed_calls.csv": "afc93422774406975a709fb1dfb9fddd5d9556fd6c3f06a15638892f05d2a824", "reports/model-refresh/2026-10-15/joon-data-reasoning-results.csv": "e1bb7f40afb0fcecc6d55a2d5cffb9d26d501514e8d665b9af2e0638744f5e28", "reports/model-refresh/2026-10-15/joon-data-reasoning-summary.csv": "bff66a545756af6a549ddceb131c43bced65524ade47036cba946fa9bbbb1225", "reports/model-refresh/2026-10-15/joon-data-speed-speed_cells_median.csv": "acd446e4d40a54cfd6bb49eb626096110ce9ac116f7cfde83ba455e44a61d9bf", "reports/model-refresh/2026-10-15/joon-data-speed-speed_effort_scaling.csv": "c3f579f70b521943073524e2f87f49c28cf345f9d10577233f43358d2e662be0" } }