{ "updated": "2026-10-15", "version": "1.1", "questionSet": "2026-06-25", "sources": [ { "id": "livebench", "name": "LiveBench", "url": "https://livebench.ai", "dataUrl": "https://livebench.ai/table_2026_06_25.csv", "date": "2026-06-25", "retrieved": "2026-10-15", "role": "23 tasks in 7 categories. Release date identifies the question set; model rows were retrieved on the snapshot date.", "kind": "Independent evaluation" }, { "id": "arena", "name": "Arena", "url": "https://arena.ai/leaderboard/text", "date": "2026-10-08", "retrieved": "2026-10-13", "role": "Human preference scores, vote counts and published uncertainty. Only explicitly matching reasoning configurations enter Consensus.", "kind": "Human preference" }, { "id": "openrouter", "name": "OpenRouter", "url": "https://openrouter.ai/api/v1/models", "date": "2026-10-15", "retrieved": "2026-10-15", "role": "Catalog API prices, context windows, modalities and declared tool support. Routed prices can differ from direct providers.", "kind": "API catalog" }, { "id": "openai", "name": "OpenAI pricing", "url": "https://developers.openai.com/api/docs/pricing", "date": "2026-10-15", "retrieved": "2026-10-15", "role": "Standard text input and output rates; long-context and service-tier premiums may apply.", "kind": "Official provider" }, { "id": "anthropic", "name": "Anthropic pricing", "url": "https://platform.claude.com/docs/en/about-claude/pricing", "date": "2026-10-15", "retrieved": "2026-10-15", "role": "Standard direct API input and output rates; caching, regional and tool fees are separate.", "kind": "Official provider" }, { "id": "google", "name": "Google pricing", "url": "https://ai.google.dev/gemini-api/docs/pricing", "date": "2026-10-15", "retrieved": "2026-10-15", "role": "Explains Gemini paid tiers, thinking-token billing and long-context pricing. Catalog prices are shown in this snapshot.", "kind": "Pricing reference" }, { "id": "bfcl", "name": "Berkeley Function Calling Leaderboard", "url": "https://gorilla.cs.berkeley.edu/leaderboard.html", "dataUrl": "https://gorilla.cs.berkeley.edu/data_overall.csv", "date": "2026-04-12", "retrieved": "2026-10-15", "role": "Separate tool-use evaluation for older model versions. Not merged into current-model ratings.", "kind": "Independent evaluation" }, { "id": "vectara", "name": "Vectara hallucination leaderboard", "url": "https://github.com/vectara/hallucination-leaderboard", "date": "2026-09-22", "retrieved": "2026-10-15", "role": "Document-summary hallucination evaluation. A different task and configuration; contextual evidence only.", "kind": "Independent evaluation" } ], "models": [ { "slug": "claude-opus-4-5", "name": "Claude Opus 4.5", "provider": "Anthropic", "setting": "high \u00b7 64K thinking", "benchmarkId": "claude-opus-4-5-20251101-thinking-64k-high-effort", "catalogId": "anthropic/claude-opus-4.5", "categories": { "Reasoning": 80.0865, "Coding": 79.654, "Agentic Coding": 39.697, "Mathematics": 90.389, "Data Analysis": 74.4417, "Language": 81.261, "IF": 62.546 }, "livebench": 72.5822, "context": 200000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-opus-4-6", "name": "Claude Opus 4.6", "provider": "Anthropic", "setting": "high \u00b7 adaptive thinking", "benchmarkId": "claude-opus-4-6-thinking-auto-high-effort", "catalogId": "anthropic/claude-opus-4.6", "categories": { "Reasoning": 88.673, "Coding": 78.1845, "Agentic Coding": 48.9897, "Mathematics": 89.317, "Data Analysis": 69.893, "Language": 83.2697, "IF": 63.3125 }, "livebench": 74.5199, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": { "score": 1504, "uncertainty": 3, "votes": 79415, "modelId": "claude-opus-4-6-high", "preliminary": false } }, { "slug": "claude-opus-4-7", "name": "Claude Opus 4.7", "provider": "Anthropic", "setting": "xhigh", "benchmarkId": "claude-opus-4-7-xhigh-effort", "catalogId": "anthropic/claude-opus-4.7", "categories": { "Reasoning": 87.1923, "Coding": 82.088, "Agentic Coding": 50.6563, "Mathematics": 92.8538, "Data Analysis": 78.2637, "Language": 77.914, "IF": 66.7375 }, "livebench": 76.5294, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "provider": "Anthropic", "setting": "medium \u00b7 adaptive thinking", "benchmarkId": "claude-sonnet-4-6-thinking-auto-medium-effort", "catalogId": "anthropic/claude-sonnet-4.6", "categories": { "Reasoning": 84.7692, "Coding": 79.2715, "Agentic Coding": 42.626, "Mathematics": 86.994, "Data Analysis": 77.946, "Language": 76.1027, "IF": 63.2208 }, "livebench": 72.99, "context": 1000000, "tools": true, "vision": true, "price": { "input": 3, "output": 15, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 3.0, "catalogOutput": 15.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-sonnet-5", "name": "Claude Sonnet 5", "provider": "Anthropic", "setting": "xhigh", "benchmarkId": "claude-sonnet-5-xhigh-effort", "catalogId": "anthropic/claude-sonnet-5", "categories": { "Reasoning": 88.6923, "Coding": 80.68, "Agentic Coding": 59.394, "Mathematics": 92.9423, "Data Analysis": 71.7407, "Language": 74.9703, "IF": 63.8585 }, "livebench": 76.0397, "context": 1000000, "tools": true, "vision": true, "price": { "input": 2, "output": 10, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "deepseek-v4-flash", "name": "DeepSeek V4 Flash", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-flash", "catalogId": "deepseek/deepseek-v4-flash", "categories": { "Reasoning": 70.5817, "Coding": 69.228, "Agentic Coding": 37.626, "Mathematics": 79.6475, "Data Analysis": 68.023, "Language": 70.124, "IF": 63.1375 }, "livebench": 65.4811, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.091, "output": 0.182, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.091, "catalogOutput": 0.182 }, "openWeights": true, "preview": false, "arena": null, "availabilityNote": "This benchmark is for the named V4 Flash configuration. The provider\u2019s unversioned V4 Flash API alias now points to V4.1 Flash; do not treat requests to that alias as measurements of this older model. Catalog offers and provider aliases must be checked separately." }, { "slug": "deepseek-v4-pro", "name": "DeepSeek V4 Pro", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-pro", "catalogId": "deepseek/deepseek-v4-pro", "categories": { "Reasoning": 82.6923, "Coding": 69.994, "Agentic Coding": 42.626, "Mathematics": 90.6755, "Data Analysis": 74.5377, "Language": 78.1303, "IF": 62.35 }, "livebench": 71.5723, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.95526, "output": 1.91052, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.95526, "catalogOutput": 1.91052 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "gemini-3-1-pro-preview", "name": "Gemini 3.1 Pro Preview", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.1-pro-preview-high", "catalogId": "google/gemini-3.1-pro-preview", "categories": { "Reasoning": 84.0048, "Coding": 76.4545, "Agentic Coding": 44.1413, "Mathematics": 91.045, "Data Analysis": 78.5413, "Language": 85.376, "IF": 79.1 }, "livebench": 76.9518, "context": 1048576, "tools": true, "vision": true, "price": { "input": 2.0, "output": 12.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 12.0 }, "openWeights": false, "preview": true, "arena": null }, { "slug": "gemini-3-5-flash", "name": "Gemini 3.5 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.5-flash-high", "catalogId": "google/gemini-3.5-flash", "categories": { "Reasoning": 82.0048, "Coding": 78.1845, "Agentic Coding": 48.9897, "Mathematics": 88.2438, "Data Analysis": 64.8573, "Language": 84.5843, "IF": 75.6 }, "livebench": 74.6378, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.5, "output": 9.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.5, "catalogOutput": 9.0 }, "openWeights": false, "preview": false, "arena": { "score": 1478, "uncertainty": 4, "votes": 48533, "modelId": "gemini-3.5-flash-high", "preliminary": false } }, { "slug": "glm-5-2", "name": "GLM-5.2", "provider": "Z.ai", "setting": "source default", "benchmarkId": "glm-5.2", "catalogId": "z-ai/glm-5.2", "categories": { "Reasoning": 78.625, "Coding": 79.654, "Agentic Coding": 51.7677, "Mathematics": 89.7812, "Data Analysis": 73.7397, "Language": 76.2417, "IF": 62.2918 }, "livebench": 73.1573, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.04, "output": 7.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.04, "catalogOutput": 7.0 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "gpt-5-2", "name": "GPT-5.2", "provider": "OpenAI", "setting": "high \u00b7 2025-12-11", "benchmarkId": "gpt-5.2-2025-12-11-high", "catalogId": "openai/gpt-5.2", "categories": { "Reasoning": 83.2115, "Coding": 76.0715, "Agentic Coding": 50.2527, "Mathematics": 93.166, "Data Analysis": 78.1633, "Language": 79.809, "IF": 61.7707 }, "livebench": 74.635, "context": 400000, "tools": true, "vision": true, "price": { "input": 1.75, "output": 14.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.75, "catalogOutput": 14.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-2-codex", "name": "GPT-5.2 Codex", "provider": "OpenAI", "setting": "source default", "benchmarkId": "gpt-5.2-codex", "catalogId": "openai/gpt-5.2-codex", "categories": { "Reasoning": 77.7115, "Coding": 83.6195, "Agentic Coding": 49.394, "Mathematics": 88.7738, "Data Analysis": 78.204, "Language": 73.6783, "IF": 66.4458 }, "livebench": 73.9753, "context": 400000, "tools": true, "vision": true, "price": { "input": 1.75, "output": 14.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.75, "catalogOutput": 14.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-4-mini", "name": "GPT-5.4 mini", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.4-mini-xhigh", "catalogId": "openai/gpt-5.4-mini", "categories": { "Reasoning": 71.3238, "Coding": 71.624, "Agentic Coding": 41.6667, "Mathematics": 78.462, "Data Analysis": 70.7857, "Language": 70.9517, "IF": 59.804 }, "livebench": 66.374, "context": 400000, "tools": true, "vision": true, "price": { "input": 0.75, "output": 4.5, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.75, "catalogOutput": 4.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-4-nano", "name": "GPT-5.4 nano", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.4-nano-xhigh", "catalogId": "openai/gpt-5.4-nano", "categories": { "Reasoning": 81.097, "Coding": 70.8385, "Agentic Coding": 46.7677, "Mathematics": 90.977, "Data Analysis": 67.6427, "Language": 62.507, "IF": 67.2048 }, "livebench": 69.5764, "context": 400000, "tools": true, "vision": true, "price": { "input": 0.2, "output": 1.25, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.2, "catalogOutput": 1.25 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-4", "name": "GPT-5.4", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.4-xhigh", "catalogId": "openai/gpt-5.4", "categories": { "Reasoning": 88.1155, "Coding": 77.5415, "Agentic Coding": 53.8383, "Mathematics": 94.148, "Data Analysis": 79.3133, "Language": 82.6337, "IF": 70.2168 }, "livebench": 77.9724, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2.5, "output": 15.0, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.5, "catalogOutput": 15.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-5", "name": "GPT-5.5", "provider": "OpenAI", "setting": "xhigh", "benchmarkId": "gpt-5.5-xhigh", "catalogId": "openai/gpt-5.5", "categories": { "Reasoning": 89.6538, "Coding": 82.1495, "Agentic Coding": 53.9897, "Mathematics": 95.8572, "Data Analysis": 81.5757, "Language": 87.363, "IF": 70.7292 }, "livebench": 80.1883, "context": 1050000, "tools": true, "vision": true, "price": { "input": 5.0, "output": 30.0, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 5.0, "catalogOutput": 30.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "grok-4-3", "name": "Grok 4.3", "provider": "xAI", "setting": "source default", "benchmarkId": "grok-4.3", "catalogId": "x-ai/grok-4.3", "categories": { "Reasoning": 70.822, "Coding": 69.9325, "Agentic Coding": 18.5353, "Mathematics": 84.339, "Data Analysis": 55.7727, "Language": 73.58, "IF": 62.75 }, "livebench": 62.2474, "context": 1000000, "tools": true, "vision": true, "price": { "input": 1.25, "output": 2.5, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.25, "catalogOutput": 2.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "grok-4-5", "name": "Grok 4.5", "provider": "xAI", "setting": "source default", "benchmarkId": "grok-4.5", "catalogId": "x-ai/grok-4.5", "categories": { "Reasoning": 87.173, "Coding": 68.5855, "Agentic Coding": 56.4647, "Mathematics": 90.8245, "Data Analysis": 73.0377, "Language": 82.7953, "IF": 71.5293 }, "livebench": 75.7729, "context": 500000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 6.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 6.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "kimi-k2-6", "name": "Kimi K2.6", "provider": "Moonshot AI", "setting": "thinking", "benchmarkId": "kimi-k2.6-thinking", "catalogId": "moonshotai/kimi-k2.6", "categories": { "Reasoning": 79.375, "Coding": 78.567, "Agentic Coding": 46.9193, "Mathematics": 84.2763, "Data Analysis": 65.1343, "Language": 75.1413, "IF": 64.3582 }, "livebench": 70.5388, "context": 262144, "tools": true, "vision": true, "price": { "input": 0.65, "output": 3.41, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.65, "catalogOutput": 3.41 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "kimi-k2-7-code", "name": "Kimi K2.7 Code", "provider": "Moonshot AI", "setting": "source default", "benchmarkId": "kimi-k2.7-code", "catalogId": "moonshotai/kimi-k2.7-code", "categories": { "Reasoning": 82.8077, "Coding": 73.959, "Agentic Coding": 45.6563, "Mathematics": 79.6043, "Data Analysis": 62.6573, "Language": 77.9057, "IF": 56.2918 }, "livebench": 68.4117, "context": 262144, "tools": true, "vision": true, "price": { "input": 0.6712, "output": 3.35, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.6712, "catalogOutput": 3.35 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "minimax-m3", "name": "MiniMax M3", "provider": "MiniMax", "setting": "source default", "benchmarkId": "minimax-m3", "catalogId": "minimax/minimax-m3", "categories": { "Reasoning": 74.4808, "Coding": 68.2025, "Agentic Coding": 40.6563, "Mathematics": 76.9475, "Data Analysis": 76.1663, "Language": 76.8357, "IF": 57.5082 }, "livebench": 67.2568, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.3, "output": 1.2, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.3, "catalogOutput": 1.2 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "qwen3-6-27b", "name": "Qwen3.6 27B", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.6-27b", "catalogId": "qwen/qwen3.6-27b", "categories": { "Reasoning": 70.2837, "Coding": 71.785, "Agentic Coding": 39.2927, "Mathematics": 79.8685, "Data Analysis": 70.4267, "Language": 63.3043, "IF": 53.229 }, "livebench": 64.0271, "context": 262144, "tools": true, "vision": true, "price": { "input": 0.32, "output": 3.2, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.32, "catalogOutput": 3.2 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "qwen3-6-plus", "name": "Qwen3.6 Plus", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.6-plus", "catalogId": "qwen/qwen3.6-plus", "categories": { "Reasoning": 75.827, "Coding": 78.1845, "Agentic Coding": 41.3637, "Mathematics": 83.7248, "Data Analysis": 69.9117, "Language": 74.9893, "IF": 58.342 }, "livebench": 68.9061, "context": 1000000, "tools": true, "vision": true, "price": { "input": 0.325, "output": 1.95, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.325, "catalogOutput": 1.95 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "qwen3-7-max", "name": "Qwen3.7 Max", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.7-max", "catalogId": "qwen/qwen3.7-max", "categories": { "Reasoning": 83.3365, "Coding": 74.219, "Agentic Coding": 43.586, "Mathematics": 85.2482, "Data Analysis": 71.786, "Language": 79.739, "IF": 74.0415 }, "livebench": 73.1366, "context": 1000000, "tools": true, "vision": false, "price": { "input": 1.475, "output": 4.425, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.475, "catalogOutput": 4.425 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-6-sol", "name": "GPT-5.6 Sol", "provider": "OpenAI", "setting": "source ID max; display xhigh", "benchmarkId": "gpt-5.6-sol-max", "catalogId": "openai/gpt-5.6-sol", "categories": { "Reasoning": 91.6538, "Coding": 83.941, "Agentic Coding": 56.212, "Mathematics": 96.1977, "Data Analysis": 79.8407, "Language": 87.6837, "IF": 71.846 }, "livebench": 81.0536, "context": 1050000, "tools": true, "vision": true, "price": { "input": 4.0, "output": 20.0, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 4.0, "catalogOutput": 20.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-6-terra", "name": "GPT-5.6 Terra", "provider": "OpenAI", "setting": "source ID max; display xhigh", "benchmarkId": "gpt-5.6-terra-max", "catalogId": "openai/gpt-5.6-terra", "categories": { "Reasoning": 90.6345, "Coding": 78.2455, "Agentic Coding": 54.9497, "Mathematics": 94.9082, "Data Analysis": 79.3067, "Language": 82.8927, "IF": 64.6165 }, "livebench": 77.9363, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 12.0, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 12.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-5-6-luna", "name": "GPT-5.6 Luna", "provider": "OpenAI", "setting": "source ID max; display xhigh", "benchmarkId": "gpt-5.6-luna-max", "catalogId": "openai/gpt-5.6-luna", "categories": { "Reasoning": 85.6442, "Coding": 82.915, "Agentic Coding": 48.4343, "Mathematics": 87.2007, "Data Analysis": 78.0333, "Language": 72.567, "IF": 60.121 }, "livebench": 73.5594, "context": 1050000, "tools": true, "vision": true, "price": { "input": 0.2, "output": 1.2, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.2, "catalogOutput": 1.2 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-fable-5", "name": "Claude Fable 5", "provider": "Anthropic", "setting": "source ID max; display xhigh", "benchmarkId": "claude-fable-5-max-effort", "catalogId": "anthropic/claude-fable-5", "categories": { "Reasoning": 89.6538, "Coding": 85.992, "Agentic Coding": 62.1717, "Mathematics": 95.9857, "Data Analysis": 80.5377, "Language": 90.684, "IF": 75.7708 }, "livebench": 82.9708, "context": 1000000, "tools": true, "vision": true, "price": { "input": 10, "output": 50, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 10.0, "catalogOutput": 50.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "muse-spark-1-1", "name": "Muse Spark 1.1", "provider": "Meta", "setting": "source ID xhigh; display high", "benchmarkId": "muse-spark-1.1-xhigh", "catalogId": "meta/muse-spark-1.1", "categories": { "Reasoning": 87.7308, "Coding": 77.1585, "Agentic Coding": 58.5353, "Mathematics": 87.1448, "Data Analysis": 72.548, "Language": 74.342, "IF": 69.6375 }, "livebench": 75.2996, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.25, "output": 4.25, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.25, "catalogOutput": 4.25 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "kimi-k3", "name": "Kimi K3", "provider": "Moonshot AI", "setting": "source default", "benchmarkId": "kimi-k3", "catalogId": "moonshotai/kimi-k3", "categories": { "Reasoning": 90.673, "Coding": 81.4455, "Agentic Coding": 62.1717, "Mathematics": 84.4368, "Data Analysis": 78.7337, "Language": 85.528, "IF": 71.3628 }, "livebench": 79.1931, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.5, "output": 15.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.5, "catalogOutput": 15.0 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "inkling", "name": "Inkling", "provider": "Thinking Machines", "setting": "xhigh", "benchmarkId": "inkling-xhigh", "catalogId": "thinkingmachines/inkling", "categories": { "Reasoning": 78.3462, "Coding": 71.0195, "Agentic Coding": 49.394, "Mathematics": 88.3648, "Data Analysis": 72.778, "Language": 73.4587, "IF": 70.0957 }, "livebench": 71.9224, "context": 524288, "tools": true, "vision": true, "price": { "input": 1.0, "output": 4.05, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.0, "catalogOutput": 4.05 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "gemini-3-5-flash-lite", "name": "Gemini 3.5 Flash-Lite", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.5-flash-lite-high", "catalogId": "google/gemini-3.5-flash-lite", "categories": { "Reasoning": 60.1875, "Coding": 76.0715, "Agentic Coding": 45.2527, "Mathematics": 73.7395, "Data Analysis": 53.2497, "Language": 71.8203, "IF": 67.2378 }, "livebench": 63.937, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.3, "output": 2.5, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.3, "catalogOutput": 2.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gemini-3-6-flash", "name": "Gemini 3.6 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.6-flash-high", "catalogId": "google/gemini-3.6-flash", "categories": { "Reasoning": 85.149, "Coding": 77.863, "Agentic Coding": 43.4343, "Mathematics": 86.4025, "Data Analysis": 62.9993, "Language": 83.8977, "IF": 75.3667 }, "livebench": 73.5875, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.75, "output": 3.75, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.75, "catalogOutput": 3.75 }, "openWeights": false, "preview": false, "arena": { "score": 1482, "uncertainty": 4, "votes": 36172, "modelId": "gemini-3.6-flash-high", "preliminary": false } }, { "slug": "claude-opus-5", "name": "Claude Opus 5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-opus-5-max-effort", "catalogId": "anthropic/claude-opus-5", "categories": { "Reasoning": 91.2115, "Coding": 81.4455, "Agentic Coding": 65.202, "Mathematics": 95.731, "Data Analysis": 74.5503, "Language": 88.688, "IF": 63.7665 }, "livebench": 80.085, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": { "score": 1489, "uncertainty": 5, "votes": 30569, "modelId": "claude-opus-5-max", "preliminary": false } }, { "slug": "claude-opus-4-8", "name": "Claude Opus 4.8", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-opus-4-8-max-effort", "catalogId": "anthropic/claude-opus-4.8", "categories": { "Reasoning": 89.1923, "Coding": 81.828, "Agentic Coding": 50.505, "Mathematics": 94.3165, "Data Analysis": 66.0333, "Language": 79.6593, "IF": 72.0335 }, "livebench": 76.224, "context": 1000000, "tools": true, "vision": true, "price": { "input": 5, "output": 25, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 5.0, "catalogOutput": 25.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash 0731", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-flash-0731", "catalogId": "deepseek/deepseek-v4-flash-0731", "categories": { "Reasoning": 86.6345, "Coding": 74.9845, "Agentic Coding": 46.7677, "Mathematics": 86.7897, "Data Analysis": 79.3273, "Language": 79.175, "IF": 65.5168 }, "livebench": 74.1708, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.016, "output": 0.64, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.016, "catalogOutput": 0.64 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "qwen3-8-max-0902", "name": "Qwen3.8 Max", "provider": "Alibaba", "setting": "source default \u00b7 version mapping unconfirmed", "benchmarkId": "qwen3.8-max", "catalogId": null, "categories": { "Reasoning": 88.2115, "Coding": 72.872, "Agentic Coding": 64.6467, "Mathematics": 91.3123, "Data Analysis": 78.413, "Language": 79.6873, "IF": 74.0835 }, "livebench": 78.4609, "context": null, "tools": null, "vision": null, "price": null, "openWeights": false, "preview": false, "arena": null }, { "slug": "muse-spark-1-2", "name": "Muse Spark 1.2", "provider": "Meta", "setting": "xhigh", "benchmarkId": "muse-spark-1.2-xhigh", "catalogId": "meta/muse-spark-1.2", "categories": { "Reasoning": 90.0048, "Coding": 77.5415, "Agentic Coding": 57.5757, "Mathematics": 91.2035, "Data Analysis": 76.4583, "Language": 78.5747, "IF": 74.325 }, "livebench": 77.9548, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.25, "output": 4.25, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.25, "catalogOutput": 4.25 }, "openWeights": false, "preview": false, "arena": { "score": 1492, "uncertainty": 8, "votes": 6118, "modelId": "muse-spark-1.2 (xHigh)", "preliminary": false } }, { "slug": "deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro 0813", "provider": "DeepSeek", "setting": "source default", "benchmarkId": "deepseek-v4-pro-0813", "catalogId": "deepseek/deepseek-v4-pro-0813", "categories": { "Reasoning": 85.8413, "Coding": 77.1585, "Agentic Coding": 54.9497, "Mathematics": 95.0863, "Data Analysis": 79.2433, "Language": 82.074, "IF": 67.7 }, "livebench": 77.4362, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.66, "output": 1.98, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.66, "catalogOutput": 1.98, "note": "This catalog quote has a published UTC time schedule. It is an observation at retrieval time, not a fixed rate for every request or day; inspect the offer before budgeting." }, "openWeights": true, "preview": false, "arena": null }, { "slug": "grok-4-6", "name": "Grok 4.6", "provider": "xAI", "setting": "source default", "benchmarkId": "grok-4.6", "catalogId": "x-ai/grok-4.6", "categories": { "Reasoning": 90.5095, "Coding": 76.776, "Agentic Coding": 57.02, "Mathematics": 92.568, "Data Analysis": 73.857, "Language": 83.697, "IF": 71.8665 }, "livebench": 78.042, "context": 500000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 6.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 6.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gemini-3-7-flash", "name": "Gemini 3.7 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.7-flash-high", "catalogId": "google/gemini-3.7-flash", "categories": { "Reasoning": 87.798, "Coding": 78.8885, "Agentic Coding": 58.283, "Mathematics": 93.468, "Data Analysis": 67.9643, "Language": 85.455, "IF": 79.9253 }, "livebench": 78.826, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.75, "output": 3.75, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.75, "catalogOutput": 3.75 }, "openWeights": false, "preview": false, "arena": { "score": 1486, "uncertainty": 5, "votes": 24173, "modelId": "gemini-3.7-flash-high", "preliminary": true } }, { "slug": "qwen3-8-27b", "name": "Qwen3.8 27B", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.8-27b", "catalogId": "qwen/qwen3.8-27b", "categories": { "Reasoning": 80.0288, "Coding": 75.689, "Agentic Coding": 61.3637, "Mathematics": 86.2127, "Data Analysis": 76.5857, "Language": 74.3457, "IF": 72.6583 }, "livebench": 75.2691, "context": 1000000, "tools": true, "vision": true, "price": { "input": 0.24, "output": 2.2, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.24, "catalogOutput": 2.2 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "deepseek-v4-flash-vision-exp", "name": "DeepSeek V4 Flash Vision Exp", "provider": "DeepSeek", "setting": "experimental \u00b7 source default", "benchmarkId": "deepseek-v4-flash-vision-exp", "catalogId": "deepseek/deepseek-v4-flash-vision-exp", "categories": { "Reasoning": 85.399, "Coding": 68.2025, "Agentic Coding": 65.101, "Mathematics": 87.8067, "Data Analysis": 79.4767, "Language": 80.3597, "IF": 70.9582 }, "livebench": 76.7577, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.2156, "output": 0.6468, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.2156, "catalogOutput": 0.6468 }, "openWeights": true, "preview": true, "arena": null }, { "slug": "glm-5-3", "name": "GLM-5.3", "provider": "Z.ai", "setting": "source default", "benchmarkId": "glm-5.3", "catalogId": "z-ai/glm-5.3", "categories": { "Reasoning": 85.803, "Coding": 78.95, "Agentic Coding": 60.909, "Mathematics": 87.898, "Data Analysis": 70.2443, "Language": 79.8563, "IF": 69.304 }, "livebench": 76.1378, "context": 1048576, "tools": true, "vision": false, "price": { "input": 0.1, "output": 4.8, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.1, "catalogOutput": 4.8 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "glm-5-3-flash", "name": "GLM-5.3 Flash", "provider": "Z.ai", "setting": "source default", "benchmarkId": "glm-5.3-flash", "catalogId": "z-ai/glm-5.3-flash", "categories": { "Reasoning": 77.6442, "Coding": 78.95, "Agentic Coding": 56.7677, "Mathematics": 81.2445, "Data Analysis": 76.401, "Language": 77.3073, "IF": 52.8208 }, "livebench": 71.5908, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.15, "output": 0.5, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.15, "catalogOutput": 0.5 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "qwen3-8-flash-next", "name": "Qwen3.8 Flash Next", "provider": "Alibaba", "setting": "source default", "benchmarkId": "qwen3.8-flash-next", "catalogId": null, "categories": { "Reasoning": 87.3798, "Coding": 72.5505, "Agentic Coding": 61.6163, "Mathematics": 85.821, "Data Analysis": 74.24, "Language": 74.643, "IF": 77.1085 }, "livebench": 76.1942, "context": null, "tools": null, "vision": null, "price": null, "openWeights": true, "preview": false, "arena": null }, { "slug": "nemotron-3-ultra-550b-a55b", "name": "Nemotron 3 Ultra", "provider": "NVIDIA", "setting": "550B A55B \u00b7 source default", "benchmarkId": "nemotron-3-ultra-550b-a55b", "catalogId": "nvidia/nemotron-3-ultra-550b-a55b", "categories": { "Reasoning": 74.697, "Coding": 70.698, "Agentic Coding": 38.7373, "Mathematics": 88.6633, "Data Analysis": 54.4567, "Language": 70.8113, "IF": 73.4455 }, "livebench": 67.3584, "context": 262144, "tools": true, "vision": false, "price": { "input": 0.6, "output": 2.4, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.6, "catalogOutput": 2.4 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "claude-fable-5-1", "name": "Claude Fable 5.1", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-fable-5-1-max-effort", "catalogId": "anthropic/claude-fable-5.1", "categories": { "Reasoning": 91.6923, "Coding": 86.375, "Agentic Coding": 66.0607, "Mathematics": 97.0067, "Data Analysis": 80.2757, "Language": 89.501, "IF": 72.9878 }, "livebench": 83.4142, "context": 1000000, "tools": true, "vision": true, "price": { "input": 10, "output": 50, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 10.0, "catalogOutput": 50.0 }, "openWeights": false, "preview": false, "arena": { "score": 1501, "uncertainty": 6, "votes": 13063, "modelId": "claude-fable-5.1-max", "preliminary": false } }, { "slug": "gemini-3-8-flash", "name": "Gemini 3.8 Flash", "provider": "Google", "setting": "high", "benchmarkId": "gemini-3.8-flash-high", "catalogId": "google/gemini-3.8-flash", "categories": { "Reasoning": 89.2933, "Coding": 72.489, "Agentic Coding": 54.2423, "Mathematics": 91.5643, "Data Analysis": 54.0057, "Language": 87.7927, "IF": 81.4125 }, "livebench": 75.8285, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.75, "output": 3.75, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.75, "catalogOutput": 3.75 }, "openWeights": false, "preview": false, "arena": { "score": 1497, "uncertainty": 5, "votes": 31437, "modelId": "gemini-3.8-flash-high", "preliminary": true } }, { "slug": "muse-spark-1-3", "name": "Muse Spark 1.3", "provider": "Meta", "setting": "xhigh", "benchmarkId": "muse-spark-1.3-xhigh", "catalogId": "meta/muse-spark-1.3", "categories": { "Reasoning": 89.6538, "Coding": 81.0625, "Agentic Coding": 64.091, "Mathematics": 95.95, "Data Analysis": 79.5653, "Language": 82.7937, "IF": 78.0042 }, "livebench": 81.5886, "context": 1048576, "tools": true, "vision": true, "price": { "input": 1.25, "output": 4.25, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 1.25, "catalogOutput": 4.25 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-astra", "name": "GPT-6 Astra", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6-astra-max", "catalogId": "openai/gpt-6-astra", "categories": { "Reasoning": 92.6538, "Coding": 80.3585, "Agentic Coding": 57.323, "Mathematics": 96.8063, "Data Analysis": 82.9733, "Language": 89.4307, "IF": 75.579 }, "livebench": 82.1607, "context": 1050000, "tools": true, "vision": true, "price": { "input": 10.0, "output": 50.0, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 10.0, "catalogOutput": 50.0 }, "openWeights": false, "preview": false, "arena": { "score": 1475, "uncertainty": 7, "votes": 10536, "modelId": "gpt-6-astra-max", "preliminary": false } }, { "slug": "deepseek-v4-1-flash", "name": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "setting": "max", "benchmarkId": "deepseek-v4.1-flash-max", "catalogId": "deepseek/deepseek-v4.1-flash", "categories": { "Reasoning": 86.6923, "Coding": 80.037, "Agentic Coding": 77.2727, "Mathematics": 93.2897, "Data Analysis": 79.2533, "Language": 81.1897, "IF": 70.0333 }, "livebench": 81.1097, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.01584, "output": 0.891, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.01584, "catalogOutput": 0.891 }, "openWeights": true, "preview": false, "arena": null }, { "slug": "grok-4-7", "name": "Grok 4.7", "provider": "xAI", "setting": "xhigh", "benchmarkId": "grok-4.7-xhigh", "catalogId": "x-ai/grok-4.7", "categories": { "Reasoning": 82.6538, "Coding": 77.1585, "Agentic Coding": 53.9897, "Mathematics": 95.678, "Data Analysis": 76.8833, "Language": 80.1387, "IF": 75.2752 }, "livebench": 77.3967, "context": 500000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 6.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 6.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "claude-opus-5-5", "name": "Claude Opus 5.5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-opus-5-5-max-effort", "catalogId": "anthropic/claude-opus-5.5", "categories": { "Reasoning": 92.1538, "Coding": 89.253, "Agentic Coding": 71.717, "Mathematics": 97.0775, "Data Analysis": 80.307, "Language": 86.27, "IF": 65.7378 }, "livebench": 83.2166, "context": 1000000, "tools": true, "vision": true, "price": { "input": 4, "output": 20, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 4.0, "catalogOutput": 20.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-sol", "name": "GPT-6 Sol", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6-sol-max", "catalogId": "openai/gpt-6-sol", "categories": { "Reasoning": 88.6538, "Coding": 81.767, "Agentic Coding": 52.8787, "Mathematics": 96.356, "Data Analysis": 81.1893, "Language": 85.2987, "IF": 68.5747 }, "livebench": 79.2455, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2.0, "output": 10.0, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-luna", "name": "GPT-6 Luna", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6-luna-max", "catalogId": "openai/gpt-6-luna", "categories": { "Reasoning": 81.7692, "Coding": 78.95, "Agentic Coding": 51.212, "Mathematics": 89.1235, "Data Analysis": 73.3697, "Language": 73.8277, "IF": 55.9333 }, "livebench": 72.0265, "context": 1050000, "tools": true, "vision": true, "price": { "input": 0.1, "output": 0.5, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.1, "catalogOutput": 0.5 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "gpt-6-1-sol", "name": "GPT-6.1 Sol", "provider": "OpenAI", "setting": "max", "benchmarkId": "gpt-6.1-sol-max", "catalogId": "openai/gpt-6.1-sol", "categories": { "Reasoning": 92.6345, "Coding": 80.3585, "Agentic Coding": 54.5453, "Mathematics": 96.8317, "Data Analysis": 82.6593, "Language": 90.126, "IF": 74.1542 }, "livebench": 81.6156, "context": 1050000, "tools": true, "vision": true, "price": { "input": 2, "output": 10, "source": "openai", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": { "score": 1484, "uncertainty": 9, "votes": 4512, "modelId": "gpt-6.1-sol-max", "preliminary": false } }, { "slug": "claude-sonnet-5-5", "name": "Claude Sonnet 5.5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-sonnet-5-5-max-effort", "catalogId": "anthropic/claude-sonnet-5.5", "categories": { "Reasoning": 91.6345, "Coding": 91.366, "Agentic Coding": 56.3133, "Mathematics": 96.1317, "Data Analysis": 59.4877, "Language": 77.984, "IF": 56.7878 }, "livebench": 75.6721, "context": 1000000, "tools": true, "vision": true, "price": { "input": 2, "output": 10, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 2.0, "catalogOutput": 10.0 }, "openWeights": false, "preview": false, "arena": null }, { "slug": "mistral-large-4-0", "name": "Mistral Large 4", "provider": "Mistral", "setting": "high", "benchmarkId": "mistral-large-4-high", "catalogId": "mistralai/mistral-large-4-0", "categories": { "Reasoning": 83.9423, "Coding": 77.1585, "Agentic Coding": 57.1717, "Mathematics": 93.5847, "Data Analysis": 76.5417, "Language": 49.6423, "IF": 64.8208 }, "livebench": 71.8374, "context": 1048576, "tools": true, "vision": true, "price": { "input": 0.68, "output": 2.09, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.68, "catalogOutput": 2.09, "note": "These are the observed routed launch rates. Mistral announced a two-week 50% discount on October 6: standard input/cache/output rates are $1.36/$0.14/$4.18 per million tokens, versus $0.68/$0.07/$2.09 during the promotion. An exact cutoff time is unverified. Check current pricing before using this preview for an ongoing budget." }, "openWeights": false, "preview": true, "arena": null, "weightsStatus": "OPEN WEIGHTS PLANNED", "officialUrl": "https://docs.mistral.ai/models/mistral-large-4-0", "availabilityNote": "This result uses the exact LiveBench high configuration on the June 25 question set. The current routed endpoint declares 1,048,576 context and 262,144 maximum output tokens; these are serving limits. No exact Arena match or same-task none/high sweep is verified.", "accessNote": "The preview API is available now. Mistral plans open weights later in October; they are not yet publicly downloadable. Confirm your selected endpoint and account access." }, { "slug": "claude-haiku-5-5", "name": "Claude Haiku 5.5", "provider": "Anthropic", "setting": "max", "benchmarkId": "claude-haiku-5-5-max-effort", "catalogId": "anthropic/claude-haiku-5.5", "categories": { "Reasoning": 84.6345, "Coding": 77.863, "Agentic Coding": 57.1717, "Mathematics": 86.435, "Data Analysis": 51.7833, "Language": 62.628, "IF": 54.6583 }, "livebench": 67.882, "context": 1000000, "tools": true, "vision": true, "price": { "input": 0.1, "output": 0.5, "source": "anthropic", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 0.1, "catalogOutput": 0.5, "contextTiers": [ { "minInput": 100000, "input": 0.5, "output": 2.5 } ], "note": "Prompts above 100,000 tokens use $0.50 input and $2.50 output per million for the full request. The base $0.10/$0.50 rates apply through 100,000 prompt tokens. These tier boundaries are verified; taxes, tools, cache writes and regional processing can add charges." }, "openWeights": false, "preview": false, "arena": null, "maxOutput": 128000, "officialUrl": "https://platform.claude.com/docs/en/models/haiku-5-5/overview", "availabilityNote": "Haiku 5.5 is available through the Claude API with 1M context and 128K standard output. Its API defaults to medium effort and adaptive thinking. Only extra high and max have matched results here; lower settings, latency and task costs are unmeasured.", "accessNote": "Migrating from Haiku 4.5 requires more than changing the model ID: replace manual thinking budgets with adaptive thinking and effort, handle thinking blocks by type, and remove sampling parameters. The newer tokenizer counts about 30% more tokens for the same text on average; use your own billed usage when comparing costs." }, { "slug": "muse-wonder", "name": "Muse Wonder", "provider": "Meta", "setting": "max \u00b7 version 1.0", "benchmarkId": "muse-wonder-1.0-max", "catalogId": "meta/muse-wonder", "categories": { "Reasoning": 90.6345, "Coding": 80.68, "Agentic Coding": 62.7273, "Mathematics": 96.4705, "Data Analysis": 81.6173, "Language": 82.9323, "IF": 73.0292 }, "livebench": 81.1559, "context": 1048576, "tools": true, "vision": true, "price": { "input": 4.0, "output": 20.0, "source": "openrouter", "retrieved": "2026-10-15", "catalogRetrieved": "2026-10-15", "catalogInput": 4.0, "catalogOutput": 20.0 }, "openWeights": false, "preview": false, "arena": null, "officialUrl": "https://dev.meta.ai/docs/models", "availabilityNote": "This result tests Muse Wonder 1.0 at max thinking on the June 25 LiveBench questions. Other thinking settings and Wonder Full have no matched result here.", "accessNote": "Wonder is available through the Meta API and Muse Code. Wonder Full requires a qualifying Muse Code subscription; choosing max effort on ordinary Wonder does not unlock Full." } ], "toolUseReference": [ { "model": "Claude-Opus-4-5-20251101 (FC)", "accuracy": 77.47, "multiTurn": 68.38, "memory": 73.76, "source": "bfcl" }, { "model": "Claude-Sonnet-4-5-20250929 (FC)", "accuracy": 73.24, "multiTurn": 61.37, "memory": 64.95, "source": "bfcl" }, { "model": "Gemini-3-Pro-Preview (Prompt)", "accuracy": 72.51, "multiTurn": 60.75, "memory": 61.72, "source": "bfcl" }, { "model": "GLM-4.6 (FC thinking)", "accuracy": 72.38, "multiTurn": 68.0, "memory": 55.7, "source": "bfcl" }, { "model": "Grok-4-1-fast-reasoning (FC)", "accuracy": 69.57, "multiTurn": 58.87, "memory": 53.98, "source": "bfcl" }, { "model": "Claude-Haiku-4-5-20251001 (FC)", "accuracy": 68.7, "multiTurn": 53.62, "memory": 54.41, "source": "bfcl" }, { "model": "Gemini-3-Pro-Preview (FC)", "accuracy": 68.14, "multiTurn": 63.12, "memory": 54.84, "source": "bfcl" }, { "model": "o3-2025-04-16 (Prompt)", "accuracy": 63.05, "multiTurn": 62.25, "memory": 51.83, "source": "bfcl" }, { "model": "Grok-4-0709 (Prompt)", "accuracy": 62.97, "multiTurn": 47.0, "memory": 50.54, "source": "bfcl" }, { "model": "Grok-4-0709 (FC)", "accuracy": 61.38, "multiTurn": 33.88, "memory": 55.91, "source": "bfcl" } ], "groundingReference": [ { "model": "openai/gpt-5.4-nano-2026-03-17", "hallucination": 3.1, "factualConsistency": 96.9, "answerRate": 100.0, "averageSummaryWords": 144.4 }, { "model": "openai/gpt-5.4-mini-2026-03-17", "hallucination": 5.5, "factualConsistency": 94.5, "answerRate": 100.0, "averageSummaryWords": 54.7 }, { "model": "openai/gpt-6-sol", "hallucination": 6.5, "factualConsistency": 93.5, "answerRate": 100.0, "averageSummaryWords": 71.4 }, { "model": "openai/gpt-5.4-2026-03-05", "hallucination": 7.0, "factualConsistency": 93.0, "answerRate": 99.9, "averageSummaryWords": 81.7 }, { "model": "openai/gpt-6-astra", "hallucination": 8.7, "factualConsistency": 91.3, "answerRate": 100.0, "averageSummaryWords": 148.5 }, { "model": "openai/gpt-5.5", "hallucination": 9.3, "factualConsistency": 90.7, "answerRate": 100.0, "averageSummaryWords": 129.6 } ], "groundingDate": "2026-09-22", "sourceHashes": { "reports/model-refresh/2026-10-15/table_2026_06_25.csv": "bf9ce6130134893d61157c214865de13d265a5e30790ff6cd4e3142a25113799", "reports/model-refresh/2026-10-15/categories_2026_06_25.json": "dad300ad18655b69db720e1b88fc5a5eac06c5b2f0e52c2bf50f10ff057674f3", "reports/model-refresh/2026-10-15/router.json": "d1f7d47808e64afa4b41fa565d64ed33ab686fac173bbf428e26b62f3c9ed5ea", "reports/model-refresh/2026-10-15/review.json": "9c546583447e58c72eeb9ccd42ee7c723655e625dd5c4982dbdd1f07b98d54fd", "reports/model-refresh/2026-10-15/bfcl-current.csv": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6" } }