{ "updated": "2026-11-04", "version": "1.1", "questionSet": "2026-06-25", "sources": [ { "id": "livebench", "name": "LiveBench", "url": "https://livebench.ai", "dataUrl": "https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv", "date": "2026-06-25", "retrieved": "2026-11-04", "role": "23 tasks in 7 categories. Release date identifies the question set; model rows were retrieved on the snapshot date.", "kind": "Independent evaluation", "currentDataUrl": "https://livebench.ai/v1/table_2026_06_25.csv", "sourceNote": "The June 25 results are unchanged. LiveBench moved the download; use the current data link below. The original URL is retained for historical reference." }, { "id": "arena", "name": "Arena", "url": "https://arena.ai/leaderboard/text", "date": "2026-10-08", "retrieved": "2026-10-13", "role": "Human preference scores, vote counts and published uncertainty. Only explicitly matching reasoning configurations enter Consensus.", "kind": "Human preference" }, { "id": "openrouter", "name": "OpenRouter", "url": "https://openrouter.ai/api/v1/models", "date": "2026-11-04", "retrieved": "2026-11-04", "role": "Catalog API prices, context windows, modalities and declared tool support. Routed prices can differ from direct providers.", "kind": "API catalog" }, { "id": "openai", "name": "OpenAI pricing", "url": "https://developers.openai.com/api/docs/pricing", "date": "2026-11-04", "retrieved": "2026-11-04", "role": "Standard text input and output rates; long-context and service-tier premiums may apply.", "kind": "Official provider" }, { "id": "anthropic", "name": "Anthropic pricing", "url": "https://platform.claude.com/docs/en/about-claude/pricing", "date": "2026-11-04", "retrieved": "2026-11-04", "role": "Standard direct API input and output rates; caching, regional and tool fees are separate.", "kind": "Official provider" }, { "id": "google", "name": "Google pricing", "url": "https://ai.google.dev/gemini-api/docs/pricing", "date": "2026-11-04", "retrieved": "2026-11-04", "role": "Explains Gemini paid tiers, thinking-token billing and long-context pricing. Catalog prices are shown in this snapshot.", "kind": "Pricing reference" }, { "id": "bfcl", "name": "Berkeley Function Calling Leaderboard", "url": "https://gorilla.cs.berkeley.edu/leaderboard.html", "dataUrl": "https://gorilla.cs.berkeley.edu/data_overall.csv", "date": "2026-04-12", "retrieved": "2026-11-04", "role": "Separate tool-use evaluation for older model versions. Not merged into current-model ratings.", "kind": "Independent evaluation" }, { "id": "vectara", "name": "Vectara hallucination leaderboard", "url": "https://github.com/vectara/hallucination-leaderboard", "date": "2026-10-22", "retrieved": "2026-11-04", "role": "Document-summary hallucination evaluation. A different task and configuration; contextual evidence only.", "kind": "Independent evaluation" } ], "models": [ { "slug": "claude-opus-4-5", "name": "Claude Opus 4.5", "provider": "Anthropic", "setting": "high \u00b7 64K thinking", "benchmarkId": "claude-opus-4-5-20251101-thinking-64k-high-effort", "catalogId": "anthropic/claude-opus-4.5", "categories": { "Reasoning": 80.0865, "Coding": 79.654, "Agentic Coding": 39.697, "Mathematics": 90.389, "Data Analysis": 74.4417, "Language": 81.261, "IF": 62.546 }, "livebench": 72.5822, "context": 200000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-opus-4-6", "name": "Claude Opus 4.6", "provider": "Anthropic", "setting": "high \u00b7 adaptive thinking", "benchmarkId": "claude-opus-4-6-thinking-auto-high-effort", "catalogId": "anthropic/claude-opus-4.6", "categories": { "Reasoning": 88.673, "Coding": 78.1845, "Agentic Coding": 48.9897, "Mathematics": 89.317, "Data Analysis": 69.893, "Language": 83.2697, "IF": 63.3125 }, "livebench": 74.5199, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": { "score": 1504, "uncertainty": 3, "votes": 79415, "modelId": "claude-opus-4-6-high", "preliminary": false } }, { "slug": "claude-opus-4-7", "name": "Claude Opus 4.7", "provider": "Anthropic", "setting": "xhigh", "benchmarkId": "claude-opus-4-7-xhigh-effort", "catalogId": "anthropic/claude-opus-4.7", "categories": { "Reasoning": 87.1923, "Coding": 82.088, "Agentic Coding": 50.6563, "Mathematics": 92.8538, "Data Analysis": 78.2637, "Language": 77.914, "IF": 66.7375 }, "livebench": 76.5294, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "provider": "Anthropic", "setting": "medium \u00b7 adaptive thinking", "benchmarkId": "claude-sonnet-4-6-thinking-auto-medium-effort", "catalogId": "anthropic/claude-sonnet-4.6", "categories": { "Reasoning": 84.7692, "Coding": 79.2715, "Agentic Coding": 42.626, "Mathematics": 86.994, "Data Analysis": 77.946, "Language": 76.1027, "IF": 63.2208 }, "livebench": 72.99, "context": 1000000, "tools": true, "vision": true, "price": { "input": 3, "output": 15, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 3.0, "catalogOutput": 15.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-sonnet-5", "name": "Claude Sonnet 5", "provider": "Anthropic", "setting": "xhigh", "benchmarkId": "claude-sonnet-5-xhigh-effort", "catalogId": "anthropic/claude-sonnet-5", "categories": { "Reasoning": 88.6923, "Coding": 80.68, "Agentic Coding": 59.394, "Mathematics": 92.9423, "Data Analysis": 71.7407, "Language": 74.9703, "IF": 63.8585 }, "livebench": 76.0397, "context": 1000000, "tools": true, "vision": true, "price": { "input": 2, "output": 10, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "deepseek-v4-flash", "name": "DeepSeek V4 Flash", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-flash", "catalogId": "deepseek/deepseek-v4-flash", "categories": { "Reasoning": 70.5817, "Coding": 69.228, "Agentic Coding": 37.626, "Mathematics": 79.6475, "Data Analysis": 68.023, "Language": 70.124, "IF": 63.1375 }, "livebench": 65.4811, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.091, "output": 0.182, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.091, "catalogOutput": 0.182 }, "openWeights": true, "preview": false, "arena": null, "availabilityNote": "This benchmark is for the named V4 Flash configuration. The provider\u2019s unversioned V4 Flash API alias now points to V4.1 Flash 1017 (October 17); do not treat requests to that alias as measurements of this older model. Catalog offers and provider aliases must be checked separately." }, { "slug": "deepseek-v4-pro", "name": "DeepSeek V4 Pro", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-pro", "catalogId": "deepseek/deepseek-v4-pro", "categories": { "Reasoning": 82.6923, "Coding": 69.994, "Agentic Coding": 42.626, "Mathematics": 90.6755, "Data Analysis": 74.5377, "Language": 78.1303, "IF": 62.35 }, "livebench": 71.5723, "context": 1048576, "tools": true, "vision": false, "price": { "input": 1.6, "output": 3.2, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.6, "catalogOutput": 3.2 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "gemini-3-1-pro-preview", "name": "Gemini 3.1 Pro Preview", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.1-pro-preview-high", "catalogId": "google/gemini-3.1-pro-preview", "categories": { "Reasoning": 84.0048, "Coding": 76.4545, "Agentic Coding": 44.1413, "Mathematics": 91.045, "Data Analysis": 78.5413, "Language": 85.376, "IF": 79.1 }, "livebench": 76.9518, "context": 1048576, "tools": true, "vision": true, "price": { "input": 2.0, "output": 12.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 12.0 }, "openWeights": false, "preview": true, "arena": null }, { "slug": "gemini-3-5-flash", "name": "Gemini 3.5 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.5-flash-high", "catalogId": "google/gemini-3.5-flash", "categories": { "Reasoning": 82.0048, "Coding": 78.1845, "Agentic Coding": 48.9897, "Mathematics": 88.2438, "Data Analysis": 64.8573, "Language": 84.5843, "IF": 75.6 }, "livebench": 74.6378, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.5, "output": 9.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.5, "catalogOutput": 9.0 }, "openWeights": false, "preview": false, "arena": { "score": 1478, "uncertainty": 4, "votes": 48533, "modelId": "gemini-3.5-flash-high", "preliminary": false } }, { "slug": "glm-5-2", "name": "GLM-5.2", "provider": "Z.ai", "setting": "source default", "benchmarkId": "glm-5.2", "catalogId": "z-ai/glm-5.2", "categories": { "Reasoning": 78.625, "Coding": 79.654, "Agentic Coding": 51.7677, "Mathematics": 89.7812, "Data Analysis": 73.7397, "Language": 76.2417, "IF": 62.2918 }, "livebench": 73.1573, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.006, "output": 6.8, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.006, "catalogOutput": 6.8 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "gpt-5-2", "name": "GPT-5.2", "provider": "OpenAI", "setting": "high \u00b7 2025-12-11", "benchmarkId": "gpt-5.2-2025-12-11-high", "catalogId": "openai/gpt-5.2", "categories": { "Reasoning": 83.2115, "Coding": 76.0715, "Agentic Coding": 50.2527, "Mathematics": 93.166, "Data Analysis": 78.1633, "Language": 79.809, "IF": 61.7707 }, "livebench": 74.635, "context": 400000, "tools": true, "vision": true, "price": { "input": 1.75, "output": 14.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.75, "catalogOutput": 14.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-2-codex", "name": "GPT-5.2 Codex", "provider": "OpenAI", "setting": "source default", "benchmarkId": "gpt-5.2-codex", "catalogId": "openai/gpt-5.2-codex", "categories": { "Reasoning": 77.7115, "Coding": 83.6195, "Agentic Coding": 49.394, "Mathematics": 88.7738, "Data Analysis": 78.204, "Language": 73.6783, "IF": 66.4458 }, "livebench": 73.9753, "context": 400000, "tools": true, "vision": true, "price": { "input": 1.75, "output": 14.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.75, "catalogOutput": 14.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-4-mini", "name": "GPT-5.4 mini", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.4-mini-xhigh", "catalogId": "openai/gpt-5.4-mini", "categories": { "Reasoning": 71.3238, "Coding": 71.624, "Agentic Coding": 41.6667, "Mathematics": 78.462, "Data Analysis": 70.7857, "Language": 70.9517, "IF": 59.804 }, "livebench": 66.374, "context": 400000, "tools": true, "vision": true, "price": { "input": 0.75, "output": 4.5, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.75, "catalogOutput": 4.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-4-nano", "name": "GPT-5.4 nano", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.4-nano-xhigh", "catalogId": "openai/gpt-5.4-nano", "categories": { "Reasoning": 81.097, "Coding": 70.8385, "Agentic Coding": 46.7677, "Mathematics": 90.977, "Data Analysis": 67.6427, "Language": 62.507, "IF": 67.2048 }, "livebench": 69.5764, "context": 400000, "tools": true, "vision": true, "price": { "input": 0.2, "output": 1.25, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.2, "catalogOutput": 1.25 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-4", "name": "GPT-5.4", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.4-xhigh", "catalogId": "openai/gpt-5.4", "categories": { "Reasoning": 88.1155, "Coding": 77.5415, "Agentic Coding": 53.8383, "Mathematics": 94.148, "Data Analysis": 79.3133, "Language": 82.6337, "IF": 70.2168 }, "livebench": 77.9724, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2.5, "output": 15.0, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.5, "catalogOutput": 15.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-5", "name": "GPT-5.5", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.5-xhigh", "catalogId": "openai/gpt-5.5", "categories": { "Reasoning": 89.6538, "Coding": 82.1495, "Agentic Coding": 53.9897, "Mathematics": 95.8572, "Data Analysis": 81.5757, "Language": 87.363, "IF": 70.7292 }, "livebench": 80.1883, "context": 1050000, "tools": true, "vision": true, "price": { "input": 5.0, "output": 30.0, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 5.0, "catalogOutput": 30.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "grok-4-3", "name": "Grok 4.3", "provider": "xAI", "setting": "source default", "benchmarkId": "grok-4.3", "catalogId": "x-ai/grok-4.3", "categories": { "Reasoning": 70.822, "Coding": 69.9325, "Agentic Coding": 18.5353, "Mathematics": 84.339, "Data Analysis": 55.7727, "Language": 73.58, "IF": 62.75 }, "livebench": 62.2474, "context": 1000000, "tools": true, "vision": true, "price": { "input": 1.25, "output": 2.5, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.25, "catalogOutput": 2.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "grok-4-5", "name": "Grok 4.5", "provider": "xAI", "setting": "source default", "benchmarkId": "grok-4.5", "catalogId": "x-ai/grok-4.5", "categories": { "Reasoning": 87.173, "Coding": 68.5855, "Agentic Coding": 56.4647, "Mathematics": 90.8245, "Data Analysis": 73.0377, "Language": 82.7953, "IF": 71.5293 }, "livebench": 75.7729, "context": 500000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 6.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 6.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "kimi-k2-6", "name": "Kimi K2.6", "provider": "Moonshot AI", "setting": "thinking", "benchmarkId": "kimi-k2.6-thinking", "catalogId": "moonshotai/kimi-k2.6", "categories": { "Reasoning": 79.375, "Coding": 78.567, "Agentic Coding": 46.9193, "Mathematics": 84.2763, "Data Analysis": 65.1343, "Language": 75.1413, "IF": 64.3582 }, "livebench": 70.5388, "context": 262144, "tools": true, "vision": true, "price": { "input": 0.65, "output": 3.41, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.65, "catalogOutput": 3.41 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "kimi-k2-7-code", "name": "Kimi K2.7 Code", "provider": "Moonshot AI", "setting": "source default", "benchmarkId": "kimi-k2.7-code", "catalogId": "moonshotai/kimi-k2.7-code", "categories": { "Reasoning": 82.8077, "Coding": 73.959, "Agentic Coding": 45.6563, "Mathematics": 79.6043, "Data Analysis": 62.6573, "Language": 77.9057, "IF": 56.2918 }, "livebench": 68.4117, "context": 262144, "tools": true, "vision": true, "price": { "input": 0.6, "output": 3.5, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.6, "catalogOutput": 3.5 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "minimax-m3", "name": "MiniMax M3", "provider": "MiniMax", "setting": "source default", "benchmarkId": "minimax-m3", "catalogId": "minimax/minimax-m3", "categories": { "Reasoning": 74.4808, "Coding": 68.2025, "Agentic Coding": 40.6563, "Mathematics": 76.9475, "Data Analysis": 76.1663, "Language": 76.8357, "IF": 57.5082 }, "livebench": 67.2568, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.3, "output": 1.2, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.3, "catalogOutput": 1.2 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "qwen3-6-27b", "name": "Qwen3.6 27B", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.6-27b", "catalogId": "qwen/qwen3.6-27b", "categories": { "Reasoning": 70.2837, "Coding": 71.785, "Agentic Coding": 39.2927, "Mathematics": 79.8685, "Data Analysis": 70.4267, "Language": 63.3043, "IF": 53.229 }, "livebench": 64.0271, "context": 262144, "tools": true, "vision": true, "price": { "input": 0.45, "output": 2.7, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.45, "catalogOutput": 2.7 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "qwen3-6-plus", "name": "Qwen3.6 Plus", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.6-plus", "catalogId": "qwen/qwen3.6-plus", "categories": { "Reasoning": 75.827, "Coding": 78.1845, "Agentic Coding": 41.3637, "Mathematics": 83.7248, "Data Analysis": 69.9117, "Language": 74.9893, "IF": 58.342 }, "livebench": 68.9061, "context": 1000000, "tools": true, "vision": true, "price": { "input": 0.325, "output": 1.95, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.325, "catalogOutput": 1.95 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "qwen3-7-max", "name": "Qwen3.7 Max", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.7-max", "catalogId": "qwen/qwen3.7-max", "categories": { "Reasoning": 83.3365, "Coding": 74.219, "Agentic Coding": 43.586, "Mathematics": 85.2482, "Data Analysis": 71.786, "Language": 79.739, "IF": 74.0415 }, "livebench": 73.1366, "context": 1000000, "tools": true, "vision": false, "price": { "input": 1.475, "output": 4.425, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.475, "catalogOutput": 4.425 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-6-sol", "name": "GPT-5.6 Sol", "provider": "OpenAI", "setting": "source ID max; display xhigh", "benchmarkId": "gpt-5.6-sol-max", "catalogId": "openai/gpt-5.6-sol", "categories": { "Reasoning": 91.6538, "Coding": 83.941, "Agentic Coding": 56.212, "Mathematics": 96.1977, "Data Analysis": 79.8407, "Language": 87.6837, "IF": 71.846 }, "livebench": 81.0536, "context": 1050000, "tools": true, "vision": true, "price": { "input": 4.0, "output": 20.0, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 4.0, "catalogOutput": 20.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-6-terra", "name": "GPT-5.6 Terra", "provider": "OpenAI", "setting": "source ID max; display xhigh", "benchmarkId": "gpt-5.6-terra-max", "catalogId": "openai/gpt-5.6-terra", "categories": { "Reasoning": 90.6345, "Coding": 78.2455, "Agentic Coding": 54.9497, "Mathematics": 94.9082, "Data Analysis": 79.3067, "Language": 82.8927, "IF": 64.6165 }, "livebench": 77.9363, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 12.0, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 12.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-6-luna", "name": "GPT-5.6 Luna", "provider": "OpenAI", "setting": "source ID max; display xhigh", "benchmarkId": "gpt-5.6-luna-max", "catalogId": "openai/gpt-5.6-luna", "categories": { "Reasoning": 85.6442, "Coding": 82.915, "Agentic Coding": 48.4343, "Mathematics": 87.2007, "Data Analysis": 78.0333, "Language": 72.567, "IF": 60.121 }, "livebench": 73.5594, "context": 1050000, "tools": true, "vision": true, "price": { "input": 0.2, "output": 1.2, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.2, "catalogOutput": 1.2 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-fable-5", "name": "Claude Fable 5", "provider": "Anthropic", "setting": "source ID max; display xhigh", "benchmarkId": "claude-fable-5-max-effort", "catalogId": "anthropic/claude-fable-5", "categories": { "Reasoning": 89.6538, "Coding": 85.992, "Agentic Coding": 62.1717, "Mathematics": 95.9857, "Data Analysis": 80.5377, "Language": 90.684, "IF": 75.7708 }, "livebench": 82.9708, "context": 1000000, "tools": true, "vision": true, "price": { "input": 10, "output": 50, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 10.0, "catalogOutput": 50.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "muse-spark-1-1", "name": "Muse Spark 1.1", "provider": "Meta", "setting": "source ID xhigh; display high", "benchmarkId": "muse-spark-1.1-xhigh", "catalogId": "meta/muse-spark-1.1", "categories": { "Reasoning": 87.7308, "Coding": 77.1585, "Agentic Coding": 58.5353, "Mathematics": 87.1448, "Data Analysis": 72.548, "Language": 74.342, "IF": 69.6375 }, "livebench": 75.2996, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.25, "output": 4.25, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.25, "catalogOutput": 4.25 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "kimi-k3", "name": "Kimi K3", "provider": "Moonshot AI", "setting": "source default", "benchmarkId": "kimi-k3", "catalogId": "moonshotai/kimi-k3", "categories": { "Reasoning": 90.673, "Coding": 81.4455, "Agentic Coding": 62.1717, "Mathematics": 84.4368, "Data Analysis": 78.7337, "Language": 85.528, "IF": 71.3628 }, "livebench": 79.1931, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.192, "output": 16.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.192, "catalogOutput": 16.0 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "inkling", "name": "Inkling", "provider": "Thinking Machines", "setting": "xhigh", "benchmarkId": "inkling-xhigh", "catalogId": "thinkingmachines/inkling", "categories": { "Reasoning": 78.3462, "Coding": 71.0195, "Agentic Coding": 49.394, "Mathematics": 88.3648, "Data Analysis": 72.778, "Language": 73.4587, "IF": 70.0957 }, "livebench": 71.9224, "context": 524288, "tools": true, "vision": true, "price": { "input": 1.0, "output": 4.05, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.0, "catalogOutput": 4.05 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "gemini-3-5-flash-lite", "name": "Gemini 3.5 Flash-Lite", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.5-flash-lite-high", "catalogId": "google/gemini-3.5-flash-lite", "categories": { "Reasoning": 60.1875, "Coding": 76.0715, "Agentic Coding": 45.2527, "Mathematics": 73.7395, "Data Analysis": 53.2497, "Language": 71.8203, "IF": 67.2378 }, "livebench": 63.937, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.3, "output": 2.5, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.3, "catalogOutput": 2.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gemini-3-6-flash", "name": "Gemini 3.6 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.6-flash-high", "catalogId": "google/gemini-3.6-flash", "categories": { "Reasoning": 85.149, "Coding": 77.863, "Agentic Coding": 43.4343, "Mathematics": 86.4025, "Data Analysis": 62.9993, "Language": 83.8977, "IF": 75.3667 }, "livebench": 73.5875, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.75, "output": 3.75, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.75, "catalogOutput": 3.75 }, "openWeights": false, "preview": false, "arena": { "score": 1482, "uncertainty": 4, "votes": 36172, "modelId": "gemini-3.6-flash-high", "preliminary": false } }, { "slug": "claude-opus-5", "name": "Claude Opus 5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-opus-5-max-effort", "catalogId": "anthropic/claude-opus-5", "categories": { "Reasoning": 91.2115, "Coding": 81.4455, "Agentic Coding": 65.202, "Mathematics": 95.731, "Data Analysis": 74.5503, "Language": 88.688, "IF": 63.7665 }, "livebench": 80.085, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": { "score": 1489, "uncertainty": 5, "votes": 30569, "modelId": "claude-opus-5-max", "preliminary": false } }, { "slug": "claude-opus-4-8", "name": "Claude Opus 4.8", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-opus-4-8-max-effort", "catalogId": "anthropic/claude-opus-4.8", "categories": { "Reasoning": 89.1923, "Coding": 81.828, "Agentic Coding": 50.505, "Mathematics": 94.3165, "Data Analysis": 66.0333, "Language": 79.6593, "IF": 72.0335 }, "livebench": 76.224, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash 0731", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-flash-0731", "catalogId": "deepseek/deepseek-v4-flash-0731", "categories": { "Reasoning": 86.6345, "Coding": 74.9845, "Agentic Coding": 46.7677, "Mathematics": 86.7897, "Data Analysis": 79.3273, "Language": 79.175, "IF": 65.5168 }, "livebench": 74.1708, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.0553, "output": 0.3, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.0553, "catalogOutput": 0.3 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "qwen3-8-max-0902", "name": "Qwen3.8 Max", "provider": "Alibaba", "setting": "source default \u00b7 version mapping unconfirmed", "benchmarkId": "qwen3.8-max", "catalogId": null, "categories": { "Reasoning": 88.2115, "Coding": 72.872, "Agentic Coding": 64.6467, "Mathematics": 91.3123, "Data Analysis": 78.413, "Language": 79.6873, "IF": 74.0835 }, "livebench": 78.4609, "context": null, "tools": null, "vision": null, "price": null, "openWeights": false, "preview": false, "arena": null }, { "slug": "muse-spark-1-2", "name": "Muse Spark 1.2", "provider": "Meta", "setting": "xhigh", "benchmarkId": "muse-spark-1.2-xhigh", "catalogId": "meta/muse-spark-1.2", "categories": { "Reasoning": 90.0048, "Coding": 77.5415, "Agentic Coding": 57.5757, "Mathematics": 91.2035, "Data Analysis": 76.4583, "Language": 78.5747, "IF": 74.325 }, "livebench": 77.9548, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.25, "output": 4.25, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.25, "catalogOutput": 4.25 }, "openWeights": false, "preview": false, "arena": { "score": 1492, "uncertainty": 8, "votes": 6118, "modelId": "muse-spark-1.2 (xHigh)", "preliminary": false } }, { "slug": "deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-pro-0813", "catalogId": "deepseek/deepseek-v4-pro-0813", "categories": { "Reasoning": 85.8413, "Coding": 77.1585, "Agentic Coding": 54.9497, "Mathematics": 95.0863, "Data Analysis": 79.2433, "Language": 82.074, "IF": 67.7 }, "livebench": 77.4362, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.115, "output": 4.6, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.115, "catalogOutput": 4.6 }, "openWeights": true, "preview": false, "arena": null, "maxOutput": 943718, "availabilityNote": "The current hosting provider lists 1,048,576 context tokens and 943,718 maximum output tokens. We use these serving limits rather than the catalog\u2019s larger general context figure. The benchmark still describes the original 0813 configuration." }, { "slug": "grok-4-6", "name": "Grok 4.6", "provider": "xAI", "setting": "source default", "benchmarkId": "grok-4.6", "catalogId": "x-ai/grok-4.6", "categories": { "Reasoning": 90.5095, "Coding": 76.776, "Agentic Coding": 57.02, "Mathematics": 92.568, "Data Analysis": 73.857, "Language": 83.697, "IF": 71.8665 }, "livebench": 78.042, "context": 500000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 6.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 6.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gemini-3-7-flash", "name": "Gemini 3.7 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.7-flash-high", "catalogId": "google/gemini-3.7-flash", "categories": { "Reasoning": 87.798, "Coding": 78.8885, "Agentic Coding": 58.283, "Mathematics": 93.468, "Data Analysis": 67.9643, "Language": 85.455, "IF": 79.9253 }, "livebench": 78.826, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.75, "output": 3.75, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.75, "catalogOutput": 3.75 }, "openWeights": false, "preview": false, "arena": { "score": 1486, "uncertainty": 5, "votes": 24173, "modelId": "gemini-3.7-flash-high", "preliminary": true } }, { "slug": "qwen3-8-27b", "name": "Qwen3.8 27B", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.8-27b", "catalogId": "qwen/qwen3.8-27b", "categories": { "Reasoning": 80.0288, "Coding": 75.689, "Agentic Coding": 61.3637, "Mathematics": 86.2127, "Data Analysis": 76.5857, "Language": 74.3457, "IF": 72.6583 }, "livebench": 75.2691, "context": 1024000, "tools": true, "vision": true, "price": { "input": 0.15, "output": 1.875, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.15, "catalogOutput": 1.875 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "deepseek-v4-flash-vision-exp", "name": "DeepSeek V4 Flash Vision Exp", "provider": "DeepSeek", "setting": "experimental \u00b7 source default", "benchmarkId": "deepseek-v4-flash-vision-exp", "catalogId": "deepseek/deepseek-v4-flash-vision-exp", "categories": { "Reasoning": 85.399, "Coding": 68.2025, "Agentic Coding": 65.101, "Mathematics": 87.8067, "Data Analysis": 79.4767, "Language": 80.3597, "IF": 70.9582 }, "livebench": 76.7577, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.2156, "output": 0.6468, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.2156, "catalogOutput": 0.6468 }, "openWeights": true, "preview": true, "arena": null }, { "slug": "glm-5-3", "name": "GLM-5.3", "provider": "Z.ai", "setting": "source default", "benchmarkId": "glm-5.3", "catalogId": "z-ai/glm-5.3", "categories": { "Reasoning": 85.803, "Coding": 78.95, "Agentic Coding": 60.909, "Mathematics": 87.898, "Data Analysis": 70.2443, "Language": 79.8563, "IF": 69.304 }, "livebench": 76.1378, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.0699, "output": 1.4, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.0699, "catalogOutput": 1.4 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "glm-5-3-flash", "name": "GLM-5.3 Flash", "provider": "Z.ai", "setting": "source default", "benchmarkId": "glm-5.3-flash", "catalogId": "z-ai/glm-5.3-flash", "categories": { "Reasoning": 77.6442, "Coding": 78.95, "Agentic Coding": 56.7677, "Mathematics": 81.2445, "Data Analysis": 76.401, "Language": 77.3073, "IF": 52.8208 }, "livebench": 71.5908, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.0269, "output": 0.0996, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.0269, "catalogOutput": 0.0996 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "qwen3-8-flash-next", "name": "Qwen3.8 Flash Next", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.8-flash-next", "catalogId": null, "categories": { "Reasoning": 87.3798, "Coding": 72.5505, "Agentic Coding": 61.6163, "Mathematics": 85.821, "Data Analysis": 74.24, "Language": 74.643, "IF": 77.1085 }, "livebench": 76.1942, "context": null, "tools": null, "vision": null, "price": null, "openWeights": true, "preview": false, "arena": null }, { "slug": "nemotron-3-ultra-550b-a55b", "name": "Nemotron 3 Ultra", "provider": "NVIDIA", "setting": "550B A55B \u00b7 source default", "benchmarkId": "nemotron-3-ultra-550b-a55b", "catalogId": "nvidia/nemotron-3-ultra-550b-a55b", "categories": { "Reasoning": 74.697, "Coding": 70.698, "Agentic Coding": 38.7373, "Mathematics": 88.6633, "Data Analysis": 54.4567, "Language": 70.8113, "IF": 73.4455 }, "livebench": 67.3584, "context": 262144, "tools": true, "vision": false, "price": { "input": 0.6, "output": 2.4, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.6, "catalogOutput": 2.4 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "claude-fable-5-1", "name": "Claude Fable 5.1", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-fable-5-1-max-effort", "catalogId": "anthropic/claude-fable-5.1", "categories": { "Reasoning": 91.6923, "Coding": 86.375, "Agentic Coding": 66.0607, "Mathematics": 97.0067, "Data Analysis": 80.2757, "Language": 89.501, "IF": 72.9878 }, "livebench": 83.4142, "context": 1000000, "tools": true, "vision": true, "price": { "input": 10, "output": 50, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 10.0, "catalogOutput": 50.0 }, "openWeights": false, "preview": false, "arena": { "score": 1501, "uncertainty": 6, "votes": 13063, "modelId": "claude-fable-5.1-max", "preliminary": false } }, { "slug": "gemini-3-8-flash", "name": "Gemini 3.8 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.8-flash-high", "catalogId": "google/gemini-3.8-flash", "categories": { "Reasoning": 89.2933, "Coding": 72.489, "Agentic Coding": 54.2423, "Mathematics": 91.5643, "Data Analysis": 54.0057, "Language": 87.7927, "IF": 81.4125 }, "livebench": 75.8285, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.825, "output": 4.125, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.825, "catalogOutput": 4.125 }, "openWeights": false, "preview": false, "arena": { "score": 1497, "uncertainty": 5, "votes": 31437, "modelId": "gemini-3.8-flash-high", "preliminary": true } }, { "slug": "muse-spark-1-3", "name": "Muse Spark 1.3", "provider": "Meta", "setting": "xhigh", "benchmarkId": "muse-spark-1.3-xhigh", "catalogId": "meta/muse-spark-1.3", "categories": { "Reasoning": 89.6538, "Coding": 81.0625, "Agentic Coding": 64.091, "Mathematics": 95.95, "Data Analysis": 79.5653, "Language": 82.7937, "IF": 78.0042 }, "livebench": 81.5886, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.25, "output": 4.25, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 1.25, "catalogOutput": 4.25 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-astra", "name": "GPT-6 Astra", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6-astra-max", "catalogId": "openai/gpt-6-astra", "categories": { "Reasoning": 92.6538, "Coding": 80.3585, "Agentic Coding": 57.323, "Mathematics": 96.8063, "Data Analysis": 82.9733, "Language": 89.4307, "IF": 75.579 }, "livebench": 82.1607, "context": 1050000, "tools": true, "vision": true, "price": { "input": 10.0, "output": 50.0, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 10.0, "catalogOutput": 50.0 }, "openWeights": false, "preview": false, "arena": { "score": 1475, "uncertainty": 7, "votes": 10536, "modelId": "gpt-6-astra-max", "preliminary": false } }, { "slug": "deepseek-v4-1-flash", "name": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "setting": "max", "benchmarkId": "deepseek-v4.1-flash-max", "catalogId": "deepseek/deepseek-v4.1-flash", "categories": { "Reasoning": 86.6923, "Coding": 80.037, "Agentic Coding": 77.2727, "Mathematics": 93.2897, "Data Analysis": 79.2533, "Language": 81.1897, "IF": 70.0333 }, "livebench": 81.1097, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.045, "output": 0.5, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.045, "catalogOutput": 0.5 }, "openWeights": true, "preview": false, "arena": null, "availabilityNote": "This result tests the earlier V4.1 Flash at max thinking. DeepSeek\u2019s native deepseek-flash name now runs the October 17 revision. These earlier results do not score that update; confirm the version your chosen hosting service runs." }, { "slug": "grok-4-7", "name": "Grok 4.7", "provider": "xAI", "setting": "xhigh", "benchmarkId": "grok-4.7-xhigh", "catalogId": "x-ai/grok-4.7", "categories": { "Reasoning": 82.6538, "Coding": 77.1585, "Agentic Coding": 53.9897, "Mathematics": 95.678, "Data Analysis": 76.8833, "Language": 80.1387, "IF": 75.2752 }, "livebench": 77.3967, "context": 500000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 6.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 6.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-opus-5-5", "name": "Claude Opus 5.5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-opus-5-5-max-effort", "catalogId": "anthropic/claude-opus-5.5", "categories": { "Reasoning": 92.1538, "Coding": 89.253, "Agentic Coding": 71.717, "Mathematics": 97.0775, "Data Analysis": 80.307, "Language": 86.27, "IF": 65.7378 }, "livebench": 83.2166, "context": 1000000, "tools": true, "vision": true, "price": { "input": 4, "output": 20, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 4.0, "catalogOutput": 20.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-sol", "name": "GPT-6 Sol", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6-sol-max", "catalogId": "openai/gpt-6-sol", "categories": { "Reasoning": 88.6538, "Coding": 81.767, "Agentic Coding": 52.8787, "Mathematics": 96.356, "Data Analysis": 81.1893, "Language": 85.2987, "IF": 68.5747 }, "livebench": 79.2455, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 10.0, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-luna", "name": "GPT-6 Luna", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6-luna-max", "catalogId": "openai/gpt-6-luna", "categories": { "Reasoning": 81.7692, "Coding": 78.95, "Agentic Coding": 51.212, "Mathematics": 89.1235, "Data Analysis": 73.3697, "Language": 73.8277, "IF": 55.9333 }, "livebench": 72.0265, "context": 1050000, "tools": true, "vision": true, "price": { "input": 0.1, "output": 0.5, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.1, "catalogOutput": 0.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-1-sol", "name": "GPT-6.1 Sol", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6.1-sol-max", "catalogId": "openai/gpt-6.1-sol", "categories": { "Reasoning": 92.6345, "Coding": 80.3585, "Agentic Coding": 54.5453, "Mathematics": 96.8317, "Data Analysis": 82.6593, "Language": 90.126, "IF": 74.1542 }, "livebench": 81.6156, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2, "output": 10, "source": "openai", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": { "score": 1484, "uncertainty": 9, "votes": 4512, "modelId": "gpt-6.1-sol-max", "preliminary": false } }, { "slug": "claude-sonnet-5-5", "name": "Claude Sonnet 5.5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-sonnet-5-5-max-effort", "catalogId": "anthropic/claude-sonnet-5.5", "categories": { "Reasoning": 91.6345, "Coding": 91.366, "Agentic Coding": 56.3133, "Mathematics": 96.1317, "Data Analysis": 59.4877, "Language": 77.984, "IF": 56.7878 }, "livebench": 75.6721, "context": 1000000, "tools": true, "vision": true, "price": { "input": 2, "output": 10, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "mistral-large-4-0", "name": "Mistral Large 4", "provider": "Mistral", "setting": "high", "benchmarkId": "mistral-large-4-high", "catalogId": "mistralai/mistral-large-4-0", "categories": { "Reasoning": 83.9423, "Coding": 77.1585, "Agentic Coding": 57.1717, "Mathematics": 93.5847, "Data Analysis": 76.5417, "Language": 49.6423, "IF": 64.8208 }, "livebench": 71.8374, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.7, "output": 2.1, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.7, "catalogOutput": 2.1, "note": "The October 30 native, OpenRouter and Gateway quotes are $0.70 input, $0.07 cached input and $2.10 output per million tokens. Historical quotes retain their original dates." }, "openWeights": true, "preview": true, "arena": null, "weightsStatus": "OPEN WEIGHTS AVAILABLE", "officialUrl": "https://docs.mistral.ai/models/mistral-large-4-0", "availabilityNote": "This result uses the exact LiveBench high configuration on the June 25 question set. The current routed endpoint declares 1,048,576 context and 262,144 maximum output tokens; these are serving limits. No exact Arena match or same-task none/high sweep is verified.", "accessNote": "Hosted access and official FP8/NVFP4 downloads are available. The October 29 weights use the Mistral Ecosystem Open-Weight License, with separate commercial licensing conditions for model-as-a-service businesses." }, { "slug": "claude-haiku-5-5", "name": "Claude Haiku 5.5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-haiku-5-5-max-effort", "catalogId": "anthropic/claude-haiku-5.5", "categories": { "Reasoning": 84.6345, "Coding": 77.863, "Agentic Coding": 57.1717, "Mathematics": 86.435, "Data Analysis": 51.7833, "Language": 62.628, "IF": 54.6583 }, "livebench": 67.882, "context": 1000000, "tools": true, "vision": true, "price": { "input": 0.1, "output": 0.5, "source": "anthropic", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.1, "catalogOutput": 0.5, "contextTiers": [ { "minInput": 100000, "input": 0.5, "output": 2.5 } ], "note": "Prompts above 100,000 tokens use $0.50 input and $2.50 output per million for the full request. The base $0.10/$0.50 rates apply through 100,000 prompt tokens. These tier boundaries are verified; taxes, tools, cache writes and regional processing can add charges." }, "openWeights": false, "preview": false, "arena": null, "maxOutput": 128000, "officialUrl": "https://platform.claude.com/docs/en/models/haiku-5-5/overview", "availabilityNote": "Haiku 5.5 is available through the Claude API with 1M context and 128K standard output. Its API defaults to medium effort and adaptive thinking. Only extra high and max have matched results here; lower settings, latency and task costs are unmeasured.", "accessNote": "Migrating from Haiku 4.5 requires more than changing the model ID: replace manual thinking budgets with adaptive thinking and effort, handle thinking blocks by type, and remove sampling parameters. The newer tokenizer counts about 30% more tokens for the same text on average; use your own billed usage when comparing costs." }, { "slug": "muse-wonder", "name": "Muse Wonder", "provider": "Meta", "setting": "max \u00b7 version 1.0", "benchmarkId": "muse-wonder-1.0-max", "catalogId": "meta/muse-wonder", "categories": { "Reasoning": 90.6345, "Coding": 80.68, "Agentic Coding": 62.7273, "Mathematics": 96.4705, "Data Analysis": 81.6173, "Language": 82.9323, "IF": 73.0292 }, "livebench": 81.1559, "context": 1048576, "tools": true, "vision": true, "price": { "input": 4.0, "output": 20.0, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 4.0, "catalogOutput": 20.0 }, "openWeights": false, "preview": false, "arena": null, "officialUrl": "https://dev.meta.ai/docs/models", "availabilityNote": "This result tests Muse Wonder 1.0 at max thinking on the June 25 LiveBench questions. Other thinking settings and Wonder Full have no matched result here.", "accessNote": "Wonder is available through the Meta API and Muse Code. Wonder Full requires a qualifying Muse Code subscription; choosing max effort on ordinary Wonder does not unlock Full. Meta currently excludes South Korea from Wonder access through both Model API and Muse Code." }, { "slug": "nemotron-3-5-lightning", "name": "Nemotron 3.5 Lightning", "provider": "NVIDIA", "setting": "source default \u00b7 thinking budget unspecified", "benchmarkId": "nemotron-3.5-lightning-30b-a3b", "catalogId": "nvidia/nemotron-3.5-lightning", "categories": { "Reasoning": 42.8702, "Coding": 45.7595, "Agentic Coding": 19.9497, "Mathematics": 66.5493, "Data Analysis": 43.0873, "Language": 38.289, "IF": 54.9417 }, "livebench": 44.4924, "context": 262144, "tools": true, "vision": false, "price": { "input": 0.07, "output": 0.2, "source": "openrouter", "retrieved": "2026-11-04", "catalogRetrieved": "2026-11-04", "catalogInput": 0.07, "catalogOutput": 0.2 }, "openWeights": true, "preview": false, "arena": null } ], "toolUseReference": [ { "model": "Claude-Opus-4-5-20251101 (FC)", "accuracy": 77.47, "multiTurn": 68.38, "memory": 73.76, "source": "bfcl" }, { "model": "Claude-Sonnet-4-5-20250929 (FC)", "accuracy": 73.24, "multiTurn": 61.37, "memory": 64.95, "source": "bfcl" }, { "model": "Gemini-3-Pro-Preview (Prompt)", "accuracy": 72.51, "multiTurn": 60.75, "memory": 61.72, "source": "bfcl" }, { "model": "GLM-4.6 (FC thinking)", "accuracy": 72.38, "multiTurn": 68.0, "memory": 55.7, "source": "bfcl" }, { "model": "Grok-4-1-fast-reasoning (FC)", "accuracy": 69.57, "multiTurn": 58.87, "memory": 53.98, "source": "bfcl" }, { "model": "Claude-Haiku-4-5-20251001 (FC)", "accuracy": 68.7, "multiTurn": 53.62, "memory": 54.41, "source": "bfcl" }, { "model": "Gemini-3-Pro-Preview (FC)", "accuracy": 68.14, "multiTurn": 63.12, "memory": 54.84, "source": "bfcl" }, { "model": "o3-2025-04-16 (Prompt)", "accuracy": 63.05, "multiTurn": 62.25, "memory": 51.83, "source": "bfcl" }, { "model": "Grok-4-0709 (Prompt)", "accuracy": 62.97, "multiTurn": 47.0, "memory": 50.54, "source": "bfcl" }, { "model": "Grok-4-0709 (FC)", "accuracy": 61.38, "multiTurn": 33.88, "memory": 55.91, "source": "bfcl" } ], "groundingReference": [ { "model": "openai/gpt-5.4-nano-2026-03-17", "hallucination": 3.1, "factualConsistency": 96.9, "answerRate": 100.0, "averageSummaryWords": 144.4 }, { "model": "openai/gpt-5.4-mini-2026-03-17", "hallucination": 5.5, "factualConsistency": 94.5, "answerRate": 100.0, "averageSummaryWords": 54.7 }, { "model": "openai/gpt-6-sol", "hallucination": 6.5, "factualConsistency": 93.5, "answerRate": 100.0, "averageSummaryWords": 71.4 }, { "model": "openai/gpt-5.4-2026-03-05", "hallucination": 7.0, "factualConsistency": 93.0, "answerRate": 99.9, "averageSummaryWords": 81.7 }, { "model": "openai/gpt-6-astra", "hallucination": 8.7, "factualConsistency": 91.3, "answerRate": 100.0, "averageSummaryWords": 148.5 }, { "model": "openai/gpt-5.5", "hallucination": 9.3, "factualConsistency": 90.7, "answerRate": 100.0, "averageSummaryWords": 129.6 } ], "groundingDate": "2026-10-22", "sourceHashes": { "reports/model-refresh/2026-11-04/table_2026_06_25.csv": "09d810986777c92b75b40a58a1829e0318e34cb5693ecae970b3145852271ba7", "reports/model-refresh/2026-11-04/categories_2026_06_25.json": "dad300ad18655b69db720e1b88fc5a5eac06c5b2f0e52c2bf50f10ff057674f3", "reports/model-refresh/2026-11-04/router.json": "27faecc4b4821bfa90af2d998afe256b72f2bde90c1e18b0691c04c7e88eb08d", "reports/model-refresh/2026-11-04/review.json": "48d4c51dab85b5b372f0296da3c138f24febb802c68fd91feb99d3be9240d1f2", "reports/model-refresh/2026-11-04/bfcl-current.csv": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6" } }