{"updated":"2026-09-23","version":"1.0","questionSet":"2026-06-25","sources":[{"id":"livebench","name":"LiveBench","url":"https://livebench.ai","dataUrl":"https://livebench.ai/table_2026_06_25.csv","date":"2026-06-25","retrieved":"2026-09-23","role":"23 tasks in 7 categories. Release date identifies the question set; model rows were retrieved on the snapshot date.","kind":"Independent evaluation"},{"id":"arena","name":"Arena","url":"https://arena.ai/leaderboard/text","date":"2026-09-13","retrieved":"2026-09-23","role":"Human preference scores, vote counts and published uncertainty. Only explicitly matching reasoning configurations enter Consensus.","kind":"Human preference"},{"id":"openrouter","name":"OpenRouter","url":"https://openrouter.ai/api/v1/models","date":"2026-09-23","retrieved":"2026-09-23","role":"Catalog API prices, context windows, modalities and declared tool support. Routed prices can differ from direct providers.","kind":"API catalog"},{"id":"openai","name":"OpenAI pricing","url":"https://developers.openai.com/api/docs/pricing","date":"2026-09-23","retrieved":"2026-09-23","role":"Standard text input and output rates; long-context and service-tier premiums may apply.","kind":"Official provider"},{"id":"anthropic","name":"Anthropic pricing","url":"https://platform.claude.com/docs/en/about-claude/pricing","date":"2026-09-23","retrieved":"2026-09-23","role":"Standard direct API input and output rates; caching, regional and tool fees are separate.","kind":"Official provider"},{"id":"google","name":"Google pricing","url":"https://ai.google.dev/gemini-api/docs/pricing","date":"2026-09-23","retrieved":"2026-09-23","role":"Explains Gemini paid tiers, thinking-token billing and long-context pricing. Catalog prices are shown in this snapshot.","kind":"Pricing reference"},{"id":"bfcl","name":"Berkeley Function Calling Leaderboard","url":"https://gorilla.cs.berkeley.edu/leaderboard.html","dataUrl":"https://gorilla.cs.berkeley.edu/data_overall.csv","date":"2026-04-12","retrieved":"2026-09-23","role":"Separate tool-use evaluation for older model versions. Not merged into current-model ratings.","kind":"Independent evaluation"},{"id":"vectara","name":"Vectara hallucination leaderboard","url":"https://github.com/vectara/hallucination-leaderboard","date":"2026-05-11","retrieved":"2026-09-23","role":"Document-summary hallucination evaluation. A different task and configuration; contextual evidence only.","kind":"Independent evaluation"}],"models":[{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","setting":"high · 64K thinking","benchmarkId":"claude-opus-4-5-20251101-thinking-64k-high-effort","catalogId":"anthropic/claude-opus-4.5","categories":{"Reasoning":80.0865,"Coding":79.654,"Agentic Coding":39.697,"Mathematics":90.389,"Data Analysis":74.4417,"Language":81.261,"IF":62.546},"livebench":72.5822,"context":200000,"tools":true,"vision":true,"price":{"input":5,"output":25,"source":"anthropic","retrieved":"2026-09-23","catalogInput":5,"catalogOutput":25},"openWeights":false,"preview":false,"arena":null},{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","setting":"high · adaptive thinking","benchmarkId":"claude-opus-4-6-thinking-auto-high-effort","catalogId":"anthropic/claude-opus-4.6","categories":{"Reasoning":88.673,"Coding":78.1845,"Agentic Coding":48.9897,"Mathematics":89.317,"Data Analysis":69.893,"Language":83.2697,"IF":63.3125},"livebench":74.5199,"context":1000000,"tools":true,"vision":true,"price":{"input":5,"output":25,"source":"anthropic","retrieved":"2026-09-23","catalogInput":5,"catalogOutput":25},"openWeights":false,"preview":false,"arena":{"score":1505,"uncertainty":4,"votes":71993,"modelId":"claude-opus-4-6-high","preliminary":false}},{"slug":"claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","setting":"xhigh","benchmarkId":"claude-opus-4-7-xhigh-effort","catalogId":"anthropic/claude-opus-4.7","categories":{"Reasoning":87.1923,"Coding":82.088,"Agentic Coding":50.6563,"Mathematics":92.8538,"Data Analysis":78.2637,"Language":77.914,"IF":66.7375},"livebench":76.5294,"context":1000000,"tools":true,"vision":true,"price":{"input":5,"output":25,"source":"anthropic","retrieved":"2026-09-23","catalogInput":5,"catalogOutput":25},"openWeights":false,"preview":false,"arena":null},{"slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","setting":"medium · adaptive thinking","benchmarkId":"claude-sonnet-4-6-thinking-auto-medium-effort","catalogId":"anthropic/claude-sonnet-4.6","categories":{"Reasoning":84.7692,"Coding":79.2715,"Agentic Coding":42.626,"Mathematics":86.994,"Data Analysis":77.946,"Language":76.1027,"IF":63.2208},"livebench":72.99,"context":1000000,"tools":true,"vision":true,"price":{"input":3,"output":15,"source":"anthropic","retrieved":"2026-09-23","catalogInput":3,"catalogOutput":15},"openWeights":false,"preview":false,"arena":null},{"slug":"claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","setting":"xhigh","benchmarkId":"claude-sonnet-5-xhigh-effort","catalogId":"anthropic/claude-sonnet-5","categories":{"Reasoning":88.6923,"Coding":80.68,"Agentic Coding":59.394,"Mathematics":92.9423,"Data Analysis":71.7407,"Language":74.9703,"IF":63.8585},"livebench":76.0397,"context":1000000,"tools":true,"vision":true,"price":{"input":2,"output":10,"source":"anthropic","retrieved":"2026-09-23","catalogInput":2,"catalogOutput":10},"openWeights":false,"preview":false,"arena":null},{"slug":"deepseek-v4-flash","name":"DeepSeek V4 Flash","provider":"DeepSeek","setting":"source default","benchmarkId":"deepseek-v4-flash","catalogId":"deepseek/deepseek-v4-flash","categories":{"Reasoning":70.5817,"Coding":69.228,"Agentic Coding":37.626,"Mathematics":79.6475,"Data Analysis":68.023,"Language":70.124,"IF":63.1375},"livebench":65.4811,"context":1048576,"tools":true,"vision":false,"price":{"input":0.088606,"output":0.177212,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.088606,"catalogOutput":0.177212},"openWeights":true,"preview":false,"arena":null},{"slug":"deepseek-v4-pro","name":"DeepSeek V4 Pro","provider":"DeepSeek","setting":"source default","benchmarkId":"deepseek-v4-pro","catalogId":"deepseek/deepseek-v4-pro","categories":{"Reasoning":82.6923,"Coding":69.994,"Agentic Coding":42.626,"Mathematics":90.6755,"Data Analysis":74.5377,"Language":78.1303,"IF":62.35},"livebench":71.5723,"context":1048576,"tools":true,"vision":false,"price":{"input":0.95526,"output":1.91052,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.95526,"catalogOutput":1.91052},"openWeights":true,"preview":false,"arena":null},{"slug":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview","provider":"Google","setting":"high","benchmarkId":"gemini-3.1-pro-preview-high","catalogId":"google/gemini-3.1-pro-preview","categories":{"Reasoning":84.0048,"Coding":76.4545,"Agentic Coding":44.1413,"Mathematics":91.045,"Data Analysis":78.5413,"Language":85.376,"IF":79.1},"livebench":76.9518,"context":1048576,"tools":true,"vision":true,"price":{"input":2,"output":12,"source":"openrouter","retrieved":"2026-09-23","catalogInput":2,"catalogOutput":12},"openWeights":false,"preview":true,"arena":null},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","setting":"high","benchmarkId":"gemini-3.5-flash-high","catalogId":"google/gemini-3.5-flash","categories":{"Reasoning":82.0048,"Coding":78.1845,"Agentic Coding":48.9897,"Mathematics":88.2438,"Data Analysis":64.8573,"Language":84.5843,"IF":75.6},"livebench":74.6378,"context":1048576,"tools":true,"vision":true,"price":{"input":1.5,"output":9,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.5,"catalogOutput":9},"openWeights":false,"preview":false,"arena":{"score":1478,"uncertainty":4,"votes":38257,"modelId":"gemini-3.5-flash-high","preliminary":false}},{"slug":"glm-5-2","name":"GLM-5.2","provider":"Z.ai","setting":"source default","benchmarkId":"glm-5.2","catalogId":"z-ai/glm-5.2","categories":{"Reasoning":78.625,"Coding":79.654,"Agentic Coding":51.7677,"Mathematics":89.7812,"Data Analysis":73.7397,"Language":76.2417,"IF":62.2918},"livebench":73.1573,"context":1048576,"tools":true,"vision":false,"price":{"input":0.6496,"output":2.0416,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.6496,"catalogOutput":2.0416},"openWeights":true,"preview":false,"arena":null},{"slug":"gpt-5-2","name":"GPT-5.2","provider":"OpenAI","setting":"high · 2025-12-11","benchmarkId":"gpt-5.2-2025-12-11-high","catalogId":"openai/gpt-5.2","categories":{"Reasoning":83.2115,"Coding":76.0715,"Agentic Coding":50.2527,"Mathematics":93.166,"Data Analysis":78.1633,"Language":79.809,"IF":61.7707},"livebench":74.635,"context":400000,"tools":true,"vision":true,"price":{"input":1.75,"output":14,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.75,"catalogOutput":14},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-2-codex","name":"GPT-5.2 Codex","provider":"OpenAI","setting":"source default","benchmarkId":"gpt-5.2-codex","catalogId":"openai/gpt-5.2-codex","categories":{"Reasoning":77.7115,"Coding":83.6195,"Agentic Coding":49.394,"Mathematics":88.7738,"Data Analysis":78.204,"Language":73.6783,"IF":66.4458},"livebench":73.9753,"context":400000,"tools":true,"vision":true,"price":{"input":1.75,"output":14,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.75,"catalogOutput":14},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-4-mini","name":"GPT-5.4 mini","provider":"OpenAI","setting":"xhigh","benchmarkId":"gpt-5.4-mini-xhigh","catalogId":"openai/gpt-5.4-mini","categories":{"Reasoning":71.3238,"Coding":71.624,"Agentic Coding":41.6667,"Mathematics":78.462,"Data Analysis":70.7857,"Language":70.9517,"IF":59.804},"livebench":66.374,"context":400000,"tools":true,"vision":true,"price":{"input":0.75,"output":4.5,"source":"openai","retrieved":"2026-09-23","catalogInput":0.75,"catalogOutput":4.5},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-4-nano","name":"GPT-5.4 nano","provider":"OpenAI","setting":"xhigh","benchmarkId":"gpt-5.4-nano-xhigh","catalogId":"openai/gpt-5.4-nano","categories":{"Reasoning":81.097,"Coding":70.8385,"Agentic Coding":46.7677,"Mathematics":90.977,"Data Analysis":67.6427,"Language":62.507,"IF":67.2048},"livebench":69.5764,"context":400000,"tools":true,"vision":true,"price":{"input":0.2,"output":1.25,"source":"openai","retrieved":"2026-09-23","catalogInput":0.2,"catalogOutput":1.25},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-4","name":"GPT-5.4","provider":"OpenAI","setting":"xhigh","benchmarkId":"gpt-5.4-xhigh","catalogId":"openai/gpt-5.4","categories":{"Reasoning":88.1155,"Coding":77.5415,"Agentic Coding":53.8383,"Mathematics":94.148,"Data Analysis":79.3133,"Language":82.6337,"IF":70.2168},"livebench":77.9724,"context":1050000,"tools":true,"vision":true,"price":{"input":2.5,"output":15,"source":"openai","retrieved":"2026-09-23","catalogInput":2.5,"catalogOutput":15},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-5","name":"GPT-5.5","provider":"OpenAI","setting":"xhigh","benchmarkId":"gpt-5.5-xhigh","catalogId":"openai/gpt-5.5","categories":{"Reasoning":89.6538,"Coding":82.1495,"Agentic Coding":53.9897,"Mathematics":95.8572,"Data Analysis":81.5757,"Language":87.363,"IF":70.7292},"livebench":80.1883,"context":1050000,"tools":true,"vision":true,"price":{"input":5,"output":30,"source":"openai","retrieved":"2026-09-23","catalogInput":5,"catalogOutput":30},"openWeights":false,"preview":false,"arena":null},{"slug":"grok-4-3","name":"Grok 4.3","provider":"xAI","setting":"source default","benchmarkId":"grok-4.3","catalogId":"x-ai/grok-4.3","categories":{"Reasoning":70.822,"Coding":69.9325,"Agentic Coding":18.5353,"Mathematics":84.339,"Data Analysis":55.7727,"Language":73.58,"IF":62.75},"livebench":62.2474,"context":1000000,"tools":true,"vision":true,"price":{"input":1.25,"output":2.5,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.25,"catalogOutput":2.5},"openWeights":false,"preview":false,"arena":null},{"slug":"grok-4-5","name":"Grok 4.5","provider":"xAI","setting":"source default","benchmarkId":"grok-4.5","catalogId":"x-ai/grok-4.5","categories":{"Reasoning":87.173,"Coding":68.5855,"Agentic Coding":56.4647,"Mathematics":90.8245,"Data Analysis":73.0377,"Language":82.7953,"IF":71.5293},"livebench":75.7729,"context":500000,"tools":true,"vision":true,"price":{"input":2,"output":6,"source":"openrouter","retrieved":"2026-09-23","catalogInput":2,"catalogOutput":6},"openWeights":false,"preview":false,"arena":null},{"slug":"kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","setting":"thinking","benchmarkId":"kimi-k2.6-thinking","catalogId":"moonshotai/kimi-k2.6","categories":{"Reasoning":79.375,"Coding":78.567,"Agentic Coding":46.9193,"Mathematics":84.2763,"Data Analysis":65.1343,"Language":75.1413,"IF":64.3582},"livebench":70.5388,"context":262144,"tools":true,"vision":true,"price":{"input":0.95,"output":4,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.95,"catalogOutput":4},"openWeights":true,"preview":false,"arena":null},{"slug":"kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","setting":"source default","benchmarkId":"kimi-k2.7-code","catalogId":"moonshotai/kimi-k2.7-code","categories":{"Reasoning":82.8077,"Coding":73.959,"Agentic Coding":45.6563,"Mathematics":79.6043,"Data Analysis":62.6573,"Language":77.9057,"IF":56.2918},"livebench":68.4117,"context":262144,"tools":true,"vision":true,"price":{"input":0.7062,"output":3.3,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.7062,"catalogOutput":3.3},"openWeights":true,"preview":false,"arena":null},{"slug":"minimax-m3","name":"MiniMax M3","provider":"MiniMax","setting":"source default","benchmarkId":"minimax-m3","catalogId":"minimax/minimax-m3","categories":{"Reasoning":74.4808,"Coding":68.2025,"Agentic Coding":40.6563,"Mathematics":76.9475,"Data Analysis":76.1663,"Language":76.8357,"IF":57.5082},"livebench":67.2568,"context":1048576,"tools":true,"vision":true,"price":{"input":0.3,"output":1.2,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.3,"catalogOutput":1.2},"openWeights":false,"preview":false,"arena":null},{"slug":"qwen3-6-27b","name":"Qwen3.6 27B","provider":"Alibaba","setting":"source default","benchmarkId":"qwen3.6-27b","catalogId":"qwen/qwen3.6-27b","categories":{"Reasoning":70.2837,"Coding":71.785,"Agentic Coding":39.2927,"Mathematics":79.8685,"Data Analysis":70.4267,"Language":63.3043,"IF":53.229},"livebench":64.0271,"context":262144,"tools":true,"vision":true,"price":{"input":0.32,"output":2.7,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.32,"catalogOutput":2.7},"openWeights":true,"preview":false,"arena":null},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","provider":"Alibaba","setting":"source default","benchmarkId":"qwen3.6-plus","catalogId":"qwen/qwen3.6-plus","categories":{"Reasoning":75.827,"Coding":78.1845,"Agentic Coding":41.3637,"Mathematics":83.7248,"Data Analysis":69.9117,"Language":74.9893,"IF":58.342},"livebench":68.9061,"context":1000000,"tools":true,"vision":true,"price":{"input":0.325,"output":1.95,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.325,"catalogOutput":1.95},"openWeights":false,"preview":false,"arena":null},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","provider":"Alibaba","setting":"source default","benchmarkId":"qwen3.7-max","catalogId":"qwen/qwen3.7-max","categories":{"Reasoning":83.3365,"Coding":74.219,"Agentic Coding":43.586,"Mathematics":85.2482,"Data Analysis":71.786,"Language":79.739,"IF":74.0415},"livebench":73.1366,"context":1000000,"tools":true,"vision":false,"price":{"input":1.475,"output":4.425,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.475,"catalogOutput":4.425},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","provider":"OpenAI","setting":"source ID max; display xhigh","benchmarkId":"gpt-5.6-sol-max","catalogId":"openai/gpt-5.6-sol","categories":{"Reasoning":91.6538,"Coding":83.941,"Agentic Coding":56.212,"Mathematics":96.1977,"Data Analysis":79.8407,"Language":87.6837,"IF":71.846},"livebench":81.0536,"context":1050000,"tools":true,"vision":true,"price":{"input":4,"output":20,"source":"openai","retrieved":"2026-09-23","catalogInput":2,"catalogOutput":10},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","provider":"OpenAI","setting":"source ID max; display xhigh","benchmarkId":"gpt-5.6-terra-max","catalogId":"openai/gpt-5.6-terra","categories":{"Reasoning":90.6345,"Coding":78.2455,"Agentic Coding":54.9497,"Mathematics":94.9082,"Data Analysis":79.3067,"Language":82.8927,"IF":64.6165},"livebench":77.9363,"context":1050000,"tools":true,"vision":true,"price":{"input":2,"output":12,"source":"openai","retrieved":"2026-09-23","catalogInput":2,"catalogOutput":12},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","provider":"OpenAI","setting":"source ID max; display xhigh","benchmarkId":"gpt-5.6-luna-max","catalogId":"openai/gpt-5.6-luna","categories":{"Reasoning":85.6442,"Coding":82.915,"Agentic Coding":48.4343,"Mathematics":87.2007,"Data Analysis":78.0333,"Language":72.567,"IF":60.121},"livebench":73.5594,"context":1050000,"tools":true,"vision":true,"price":{"input":0.2,"output":1.2,"source":"openai","retrieved":"2026-09-23","catalogInput":0.2,"catalogOutput":1.2},"openWeights":false,"preview":false,"arena":null},{"slug":"claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","setting":"source ID max; display xhigh","benchmarkId":"claude-fable-5-max-effort","catalogId":"anthropic/claude-fable-5","categories":{"Reasoning":89.6538,"Coding":85.992,"Agentic Coding":62.1717,"Mathematics":95.9857,"Data Analysis":80.5377,"Language":90.684,"IF":75.7708},"livebench":82.9708,"context":1000000,"tools":true,"vision":true,"price":{"input":10,"output":50,"source":"anthropic","retrieved":"2026-09-23","catalogInput":10,"catalogOutput":50},"openWeights":false,"preview":false,"arena":null},{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","setting":"source ID xhigh; display high","benchmarkId":"muse-spark-1.1-xhigh","catalogId":"meta/muse-spark-1.1","categories":{"Reasoning":87.7308,"Coding":77.1585,"Agentic Coding":58.5353,"Mathematics":87.1448,"Data Analysis":72.548,"Language":74.342,"IF":69.6375},"livebench":75.2996,"context":1048576,"tools":true,"vision":true,"price":{"input":1.25,"output":4.25,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.25,"catalogOutput":4.25},"openWeights":false,"preview":false,"arena":null},{"slug":"kimi-k3","name":"Kimi K3","provider":"Moonshot AI","setting":"source default","benchmarkId":"kimi-k3","catalogId":"moonshotai/kimi-k3","categories":{"Reasoning":90.673,"Coding":81.4455,"Agentic Coding":62.1717,"Mathematics":84.4368,"Data Analysis":78.7337,"Language":85.528,"IF":71.3628},"livebench":79.1931,"context":1048576,"tools":true,"vision":true,"price":{"input":3,"output":15,"source":"openrouter","retrieved":"2026-09-23","catalogInput":3,"catalogOutput":15},"openWeights":true,"preview":false,"arena":null},{"slug":"inkling","name":"Inkling","provider":"Thinking Machines","setting":"xhigh","benchmarkId":"inkling-xhigh","catalogId":"thinkingmachines/inkling","categories":{"Reasoning":78.3462,"Coding":71.0195,"Agentic Coding":49.394,"Mathematics":88.3648,"Data Analysis":72.778,"Language":73.4587,"IF":70.0957},"livebench":71.9224,"context":1048576,"tools":true,"vision":true,"price":{"input":1,"output":4.05,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1,"catalogOutput":4.05},"openWeights":true,"preview":false,"arena":null},{"slug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","provider":"Google","setting":"high","benchmarkId":"gemini-3.5-flash-lite-high","catalogId":"google/gemini-3.5-flash-lite","categories":{"Reasoning":60.1875,"Coding":76.0715,"Agentic Coding":45.2527,"Mathematics":73.7395,"Data Analysis":53.2497,"Language":71.8203,"IF":67.2378},"livebench":63.937,"context":1048576,"tools":true,"vision":true,"price":{"input":0.3,"output":2.5,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.3,"catalogOutput":2.5},"openWeights":false,"preview":false,"arena":null},{"slug":"gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","setting":"high","benchmarkId":"gemini-3.6-flash-high","catalogId":"google/gemini-3.6-flash","categories":{"Reasoning":85.149,"Coding":77.863,"Agentic Coding":43.4343,"Mathematics":86.4025,"Data Analysis":62.9993,"Language":83.8977,"IF":75.3667},"livebench":73.5875,"context":1048576,"tools":true,"vision":true,"price":{"input":0.75,"output":3.75,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.75,"catalogOutput":3.75},"openWeights":false,"preview":false,"arena":{"score":1480,"uncertainty":5,"votes":26445,"modelId":"gemini-3.6-flash-high","preliminary":false}},{"slug":"claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","setting":"max","benchmarkId":"claude-opus-5-max-effort","catalogId":"anthropic/claude-opus-5","categories":{"Reasoning":91.2115,"Coding":81.4455,"Agentic Coding":65.202,"Mathematics":95.731,"Data Analysis":74.5503,"Language":88.688,"IF":63.7665},"livebench":80.085,"context":1000000,"tools":true,"vision":true,"price":{"input":5,"output":25,"source":"anthropic","retrieved":"2026-09-23","catalogInput":5,"catalogOutput":25},"openWeights":false,"preview":false,"arena":{"score":1487,"uncertainty":5,"votes":20706,"modelId":"claude-opus-5-max","preliminary":false}},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","setting":"max","benchmarkId":"claude-opus-4-8-max-effort","catalogId":"anthropic/claude-opus-4.8","categories":{"Reasoning":89.1923,"Coding":81.828,"Agentic Coding":50.505,"Mathematics":94.3165,"Data Analysis":66.0333,"Language":79.6593,"IF":72.0335},"livebench":76.224,"context":1000000,"tools":true,"vision":true,"price":{"input":5,"output":25,"source":"anthropic","retrieved":"2026-09-23","catalogInput":5,"catalogOutput":25},"openWeights":false,"preview":false,"arena":null},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","provider":"DeepSeek","setting":"source default","benchmarkId":"deepseek-v4-flash-0731","catalogId":"deepseek/deepseek-v4-flash-0731","categories":{"Reasoning":86.6345,"Coding":74.9845,"Agentic Coding":46.7677,"Mathematics":86.7897,"Data Analysis":79.3273,"Language":79.175,"IF":65.5168},"livebench":74.1708,"context":1310720,"tools":true,"vision":false,"price":{"input":0.04,"output":0.64,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.04,"catalogOutput":0.64},"openWeights":true,"preview":false,"arena":null},{"slug":"qwen3-8-max-0902","name":"Qwen3.8 Max","provider":"Alibaba","setting":"source default · version mapping unconfirmed","benchmarkId":"qwen3.8-max","catalogId":null,"categories":{"Reasoning":88.2115,"Coding":72.872,"Agentic Coding":64.6467,"Mathematics":91.3123,"Data Analysis":78.413,"Language":79.6873,"IF":74.0835},"livebench":78.4609,"context":null,"tools":null,"vision":null,"price":null,"openWeights":false,"preview":false,"arena":null},{"slug":"muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","setting":"xhigh","benchmarkId":"muse-spark-1.2-xhigh","catalogId":"meta/muse-spark-1.2","categories":{"Reasoning":90.0048,"Coding":77.5415,"Agentic Coding":57.5757,"Mathematics":91.2035,"Data Analysis":76.4583,"Language":78.5747,"IF":74.325},"livebench":77.9548,"context":1048576,"tools":true,"vision":true,"price":{"input":1.25,"output":4.25,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.25,"catalogOutput":4.25},"openWeights":false,"preview":false,"arena":{"score":1500,"uncertainty":11,"votes":3227,"modelId":"muse-spark-1.2 (xHigh)","preliminary":false}},{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","provider":"DeepSeek","setting":"source default","benchmarkId":"deepseek-v4-pro-0813","catalogId":"deepseek/deepseek-v4-pro-0813","categories":{"Reasoning":85.8413,"Coding":77.1585,"Agentic Coding":54.9497,"Mathematics":95.0863,"Data Analysis":79.2433,"Language":82.074,"IF":67.7},"livebench":77.4362,"context":1048576,"tools":true,"vision":false,"price":{"input":0.66,"output":1.98,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.66,"catalogOutput":1.98},"openWeights":true,"preview":false,"arena":null},{"slug":"grok-4-6","name":"Grok 4.6","provider":"xAI","setting":"source default","benchmarkId":"grok-4.6","catalogId":"x-ai/grok-4.6","categories":{"Reasoning":90.5095,"Coding":76.776,"Agentic Coding":57.02,"Mathematics":92.568,"Data Analysis":73.857,"Language":83.697,"IF":71.8665},"livebench":78.042,"context":500000,"tools":true,"vision":true,"price":{"input":2,"output":6,"source":"openrouter","retrieved":"2026-09-23","catalogInput":2,"catalogOutput":6},"openWeights":false,"preview":false,"arena":null},{"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","setting":"high","benchmarkId":"gemini-3.7-flash-high","catalogId":"google/gemini-3.7-flash","categories":{"Reasoning":87.798,"Coding":78.8885,"Agentic Coding":58.283,"Mathematics":93.468,"Data Analysis":67.9643,"Language":85.455,"IF":79.9253},"livebench":78.826,"context":1048576,"tools":true,"vision":true,"price":{"input":0.75,"output":3.75,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.75,"catalogOutput":3.75},"openWeights":false,"preview":false,"arena":{"score":1490,"uncertainty":8,"votes":5640,"modelId":"gemini-3.7-flash-high","preliminary":true}},{"slug":"qwen3-8-27b","name":"Qwen3.8 27B","provider":"Alibaba","setting":"source default","benchmarkId":"qwen3.8-27b","catalogId":"qwen/qwen3.8-27b","categories":{"Reasoning":80.0288,"Coding":75.689,"Agentic Coding":61.3637,"Mathematics":86.2127,"Data Analysis":76.5857,"Language":74.3457,"IF":72.6583},"livebench":75.2691,"context":1000000,"tools":true,"vision":true,"price":{"input":0.42,"output":3,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.42,"catalogOutput":3},"openWeights":true,"preview":false,"arena":null},{"slug":"deepseek-v4-flash-vision-exp","name":"DeepSeek V4 Flash Vision Exp","provider":"DeepSeek","setting":"experimental · source default","benchmarkId":"deepseek-v4-flash-vision-exp","catalogId":"deepseek/deepseek-v4-flash-vision-exp","categories":{"Reasoning":85.399,"Coding":68.2025,"Agentic Coding":65.101,"Mathematics":87.8067,"Data Analysis":79.4767,"Language":80.3597,"IF":70.9582},"livebench":76.7577,"context":1048576,"tools":true,"vision":true,"price":{"input":0.22,"output":0.66,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.22,"catalogOutput":0.66},"openWeights":true,"preview":true,"arena":null},{"slug":"glm-5-3","name":"GLM-5.3","provider":"Z.ai","setting":"source default","benchmarkId":"glm-5.3","catalogId":"z-ai/glm-5.3","categories":{"Reasoning":85.803,"Coding":78.95,"Agentic Coding":60.909,"Mathematics":87.898,"Data Analysis":70.2443,"Language":79.8563,"IF":69.304},"livebench":76.1378,"context":1310720,"tools":true,"vision":false,"price":{"input":0.84,"output":2.64,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.84,"catalogOutput":2.64},"openWeights":true,"preview":false,"arena":null},{"slug":"glm-5-3-flash","name":"GLM-5.3 Flash","provider":"Z.ai","setting":"source default","benchmarkId":"glm-5.3-flash","catalogId":"z-ai/glm-5.3-flash","categories":{"Reasoning":77.6442,"Coding":78.95,"Agentic Coding":56.7677,"Mathematics":81.2445,"Data Analysis":76.401,"Language":77.3073,"IF":52.8208},"livebench":71.5908,"context":1310720,"tools":true,"vision":true,"price":{"input":0.15,"output":0.5,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.15,"catalogOutput":0.5},"openWeights":true,"preview":false,"arena":null},{"slug":"qwen3-8-flash-next","name":"Qwen3.8 Flash Next","provider":"Alibaba","setting":"source default","benchmarkId":"qwen3.8-flash-next","catalogId":null,"categories":{"Reasoning":87.3798,"Coding":72.5505,"Agentic Coding":61.6163,"Mathematics":85.821,"Data Analysis":74.24,"Language":74.643,"IF":77.1085},"livebench":76.1942,"context":null,"tools":null,"vision":null,"price":null,"openWeights":true,"preview":false,"arena":null},{"slug":"nemotron-3-ultra-550b-a55b","name":"Nemotron 3 Ultra","provider":"NVIDIA","setting":"550B A55B · source default","benchmarkId":"nemotron-3-ultra-550b-a55b","catalogId":"nvidia/nemotron-3-ultra-550b-a55b","categories":{"Reasoning":74.697,"Coding":70.698,"Agentic Coding":38.7373,"Mathematics":88.6633,"Data Analysis":54.4567,"Language":70.8113,"IF":73.4455},"livebench":67.3584,"context":262144,"tools":true,"vision":false,"price":{"input":0.6,"output":2.4,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.6,"catalogOutput":2.4},"openWeights":true,"preview":false,"arena":null},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","setting":"max","benchmarkId":"claude-fable-5-1-max-effort","catalogId":"anthropic/claude-fable-5.1","categories":{"Reasoning":91.6923,"Coding":86.375,"Agentic Coding":66.0607,"Mathematics":97.0067,"Data Analysis":80.2757,"Language":89.501,"IF":72.9878},"livebench":83.4142,"context":1000000,"tools":true,"vision":true,"price":{"input":10,"output":50,"source":"anthropic","retrieved":"2026-09-23","catalogInput":10,"catalogOutput":50},"openWeights":false,"preview":false,"arena":{"score":1498,"uncertainty":8,"votes":5783,"modelId":"claude-fable-5.1-max","preliminary":false}},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","setting":"high","benchmarkId":"gemini-3.8-flash-high","catalogId":"google/gemini-3.8-flash","categories":{"Reasoning":89.2933,"Coding":72.489,"Agentic Coding":54.2423,"Mathematics":91.5643,"Data Analysis":54.0057,"Language":87.7927,"IF":81.4125},"livebench":75.8285,"context":1048576,"tools":true,"vision":true,"price":{"input":0.75,"output":3.75,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.75,"catalogOutput":3.75},"openWeights":false,"preview":false,"arena":{"score":1493,"uncertainty":9,"votes":5076,"modelId":"gemini-3.8-flash-high","preliminary":true}},{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","setting":"xhigh","benchmarkId":"muse-spark-1.3-xhigh","catalogId":"meta/muse-spark-1.3","categories":{"Reasoning":89.6538,"Coding":81.0625,"Agentic Coding":64.091,"Mathematics":95.95,"Data Analysis":79.5653,"Language":82.7937,"IF":78.0042},"livebench":81.5886,"context":1048576,"tools":true,"vision":true,"price":{"input":1.25,"output":4.25,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.25,"catalogOutput":4.25},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-6-astra","name":"GPT-6 Astra","provider":"OpenAI","setting":"max","benchmarkId":"gpt-6-astra-max","catalogId":"openai/gpt-6-astra","categories":{"Reasoning":92.6538,"Coding":80.3585,"Agentic Coding":57.323,"Mathematics":96.8063,"Data Analysis":82.9733,"Language":89.4307,"IF":75.579},"livebench":82.1607,"context":1050000,"tools":true,"vision":true,"price":{"input":10,"output":50,"source":"openai","retrieved":"2026-09-23","catalogInput":10,"catalogOutput":50},"openWeights":false,"preview":false,"arena":{"score":1480,"uncertainty":12,"votes":2693,"modelId":"gpt-6-astra-max","preliminary":false}},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","setting":"max","benchmarkId":"deepseek-v4.1-flash-max","catalogId":"deepseek/deepseek-v4.1-flash","categories":{"Reasoning":86.6923,"Coding":80.037,"Agentic Coding":77.2727,"Mathematics":93.2897,"Data Analysis":79.2533,"Language":81.1897,"IF":70.0333},"livebench":81.1097,"context":1048576,"tools":true,"vision":true,"price":{"input":0.094,"output":0.6,"source":"openrouter","retrieved":"2026-09-23","catalogInput":0.094,"catalogOutput":0.6},"openWeights":true,"preview":false,"arena":null},{"slug":"grok-4-7","name":"Grok 4.7","provider":"xAI","setting":"xhigh","benchmarkId":"grok-4.7-xhigh","catalogId":"x-ai/grok-4.7","categories":{"Reasoning":82.6538,"Coding":77.1585,"Agentic Coding":53.9897,"Mathematics":95.678,"Data Analysis":76.8833,"Language":80.1387,"IF":75.2752},"livebench":77.3967,"context":500000,"tools":true,"vision":true,"price":{"input":1.6,"output":4.8,"source":"openrouter","retrieved":"2026-09-23","catalogInput":1.6,"catalogOutput":4.8},"openWeights":false,"preview":false,"arena":null},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","setting":"max","benchmarkId":"claude-opus-5-5-max-effort","catalogId":"anthropic/claude-opus-5.5","categories":{"Reasoning":92.1538,"Coding":89.253,"Agentic Coding":71.717,"Mathematics":97.0775,"Data Analysis":80.307,"Language":86.27,"IF":65.7378},"livebench":83.2166,"context":1000000,"tools":true,"vision":true,"price":{"input":4,"output":20,"source":"anthropic","retrieved":"2026-09-23","catalogInput":4,"catalogOutput":20},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-6-sol","name":"GPT-6 Sol","provider":"OpenAI","setting":"max","benchmarkId":"gpt-6-sol-max","catalogId":"openai/gpt-6-sol","categories":{"Reasoning":88.6538,"Coding":81.767,"Agentic Coding":52.8787,"Mathematics":96.356,"Data Analysis":81.1893,"Language":85.2987,"IF":68.5747},"livebench":79.2455,"context":1050000,"tools":true,"vision":true,"price":{"input":2,"output":10,"source":"openai","retrieved":"2026-09-23","catalogInput":2,"catalogOutput":10},"openWeights":false,"preview":false,"arena":null},{"slug":"gpt-6-luna","name":"GPT-6 Luna","provider":"OpenAI","setting":"max","benchmarkId":"gpt-6-luna-max","catalogId":"openai/gpt-6-luna","categories":{"Reasoning":81.7692,"Coding":78.95,"Agentic Coding":51.212,"Mathematics":89.1235,"Data Analysis":73.3697,"Language":73.8277,"IF":55.9333},"livebench":72.0265,"context":1050000,"tools":true,"vision":true,"price":{"input":0.1,"output":0.5,"source":"openai","retrieved":"2026-09-23","catalogInput":0.1,"catalogOutput":0.5},"openWeights":false,"preview":false,"arena":null}],"toolUseReference":[{"model":"Claude-Opus-4-5-20251101 (FC)","accuracy":77.47,"multiTurn":68.38,"memory":73.76,"source":"bfcl"},{"model":"Claude-Sonnet-4-5-20250929 (FC)","accuracy":73.24,"multiTurn":61.37,"memory":64.95,"source":"bfcl"},{"model":"Gemini-3-Pro-Preview (Prompt)","accuracy":72.51,"multiTurn":60.75,"memory":61.72,"source":"bfcl"},{"model":"GLM-4.6 (FC thinking)","accuracy":72.38,"multiTurn":68,"memory":55.7,"source":"bfcl"},{"model":"Grok-4-1-fast-reasoning (FC)","accuracy":69.57,"multiTurn":58.87,"memory":53.98,"source":"bfcl"},{"model":"Claude-Haiku-4-5-20251001 (FC)","accuracy":68.7,"multiTurn":53.62,"memory":54.41,"source":"bfcl"},{"model":"Gemini-3-Pro-Preview (FC)","accuracy":68.14,"multiTurn":63.12,"memory":54.84,"source":"bfcl"},{"model":"o3-2025-04-16 (Prompt)","accuracy":63.05,"multiTurn":62.25,"memory":51.83,"source":"bfcl"},{"model":"Grok-4-0709 (Prompt)","accuracy":62.97,"multiTurn":47,"memory":50.54,"source":"bfcl"},{"model":"Grok-4-0709 (FC)","accuracy":61.38,"multiTurn":33.88,"memory":55.91,"source":"bfcl"}],"sourceHashes":{"table_2026_06_25.csv":"05f59189a82813c446681b0c5902a559ef9c5d55263a2bb0ae2d1868fb5ecb2d","categories_2026_06_25.json":"dad300ad18655b69db720e1b88fc5a5eac06c5b2f0e52c2bf50f10ff057674f3","router.raw":"6e4f855eddf52e54bfe29692fbf2879ec6e5bf25e6bcc5b5209f9651b6dda7db","bfcl.csv":"ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6"}}