{ "generated_at": "2026-11-06T21:33:47Z", "version": 1, "labs": [ { "id": "openai", "name": "OpenAI", "short": "OpenAI", "color": "#10A37F", "color_note": "Historic ChatGPT/OpenAI brand green (#10A37F). The current wordmark is black; green is kept so the line stays legible next to xAI.", "website": "https://openai.com", "sources": [ { "label": "News RSS", "url": "https://openai.com/news/rss.xml", "kind": "rss" }, { "label": "News", "url": "https://openai.com/news/", "kind": "html" } ], "flagship_hints": [ "^GPT-\\d", "^o\\d", "GPT-\\d+(\\.\\d+)?(?!.*(mini|nano))", "^OpenAI o\\d" ] }, { "id": "anthropic", "name": "Anthropic", "short": "Anthropic", "color": "#D97757", "color_note": "Anthropic/Claude terracotta (#D97757), official brand colour.", "website": "https://www.anthropic.com", "sources": [ { "label": "News", "url": "https://www.anthropic.com/news", "kind": "html" }, { "label": "Model overview (docs)", "url": "https://docs.anthropic.com/en/docs/about-claude/models/overview", "kind": "html" } ], "flagship_hints": [ "Claude (Opus|Fable|Mythos)", "Claude \\d(\\.\\d)? Opus", "^Claude (1\\.3|2|2\\.1|3\\.5 Sonnet|3\\.7 Sonnet)$" ], "name_prefixes": [ { "match": "^(opus|sonnet|haiku|fable|mythos)\\b", "prefix": "Claude " } ] }, { "id": "google", "name": "Google DeepMind", "short": "Google", "color": "#9168C0", "color_note": "Gemini brand-gradient purple (#9168C0). The four-colour Google palette cannot represent a single line.", "website": "https://deepmind.google", "sources": [ { "label": "Google Keyword blog — Gemini RSS", "url": "https://blog.google/products-and-platforms/products/gemini/rss/", "kind": "rss" }, { "label": "DeepMind blog", "url": "https://deepmind.google/blog/", "kind": "html" }, { "label": "Gemini models docs", "url": "https://ai.google.dev/gemini-api/docs/models", "kind": "html" } ], "flagship_hints": [ "Gemini \\d(\\.\\d)? (Pro|Ultra)", "Gemini \\d(\\.\\d)? Deep Think", "^PaLM \\d", "^Gemini \\d(\\.\\d)?$" ] }, { "id": "xai", "name": "xAI", "short": "xAI", "color": "#111111", "color_note": "xAI/Grok brand is monochrome black; used as-is.", "website": "https://x.ai", "sources": [ { "label": "News", "url": "https://x.ai/news", "kind": "html" }, { "label": "API models docs", "url": "https://docs.x.ai/docs/models", "kind": "html" } ], "flagship_hints": [ "Grok \\d(\\.\\d)?(?!.*(mini|fast))", "^Grok-\\d(\\.\\d)?$" ] }, { "id": "meta", "name": "Meta", "short": "Meta", "color": "#0866FF", "color_note": "Meta brand blue (#0866FF), official.", "website": "https://ai.meta.com", "sources": [ { "label": "Meta AI blog", "url": "https://ai.meta.com/blog/", "kind": "html" }, { "label": "Hugging Face org", "url": "https://huggingface.co/api/models?author=meta-llama&sort=lastModified&limit=20", "kind": "hf-org" } ], "flagship_hints": [ "Llama \\d", "Muse Spark", "Meta Superintelligence" ] }, { "id": "deepseek", "name": "DeepSeek", "short": "DeepSeek", "color": "#4D6BFE", "color_note": "DeepSeek whale-logo blue (#4D6BFE), official.", "website": "https://www.deepseek.com", "sources": [ { "label": "API news", "url": "https://api-docs.deepseek.com/news/", "kind": "html" }, { "label": "Hugging Face org", "url": "https://huggingface.co/api/models?author=deepseek-ai&sort=lastModified&limit=20", "kind": "hf-org" } ], "flagship_hints": [ "DeepSeek-V\\d", "DeepSeek-R\\d", "^DeepSeek LLM" ] }, { "id": "alibaba", "name": "Alibaba (Qwen)", "short": "Qwen", "color": "#FF6A00", "color_note": "Alibaba Cloud orange (#FF6A00), official corporate colour; Qwen purple would collide with Gemini.", "website": "https://qwen.ai", "sources": [ { "label": "Qwen blog", "url": "https://qwenlm.github.io/blog/", "kind": "html" }, { "label": "Alibaba Cloud press room", "url": "https://www.alibabacloud.com/en/press-room", "kind": "html" }, { "label": "Hugging Face org", "url": "https://huggingface.co/api/models?author=Qwen&sort=lastModified&limit=20", "kind": "hf-org" } ], "flagship_hints": [ "Qwen\\d(\\.\\d)?-?(Max|Plus)?", "Qwen \\d(\\.\\d)? Max", "^Qwen-\\d+B$" ] }, { "id": "moonshot", "name": "Moonshot AI (Kimi)", "short": "Kimi", "color": "#06B6D4", "color_note": "Approximate: Moonshot/Kimi brand is monochrome; cyan chosen for separation from the blues and greens.", "website": "https://www.moonshot.ai", "sources": [ { "label": "Moonshot site", "url": "https://www.moonshot.ai/", "kind": "html" }, { "label": "Hugging Face org", "url": "https://huggingface.co/api/models?author=moonshotai&sort=lastModified&limit=20", "kind": "hf-org" } ], "flagship_hints": [ "Kimi K\\d(\\.\\d)?", "^Kimi Chat$" ], "name_prefixes": [ { "match": "^(k[0-9])\\b", "prefix": "Kimi " } ] }, { "id": "zhipu", "name": "Zhipu AI (Z.ai)", "short": "GLM", "color": "#DB2777", "color_note": "Approximate: Z.ai brand is monochrome; magenta chosen for separation.", "website": "https://z.ai", "sources": [ { "label": "Z.ai release notes (new releases)", "url": "https://docs.z.ai/release-notes/new-released", "kind": "html" }, { "label": "Release notes", "url": "https://docs.z.ai/release-notes", "kind": "html" }, { "label": "Hugging Face org", "url": "https://huggingface.co/api/models?author=zai-org&sort=lastModified&limit=20", "kind": "hf-org" } ], "flagship_hints": [ "GLM-\\d(\\.\\d)?(?!.*(Air|Flash))" ] }, { "id": "mistral", "name": "Mistral AI", "short": "Mistral", "color": "#E10500", "color_note": "Red end of the Mistral yellow–orange–red gradient (#E10500); yellow and orange are reserved for predictions and Qwen.", "website": "https://mistral.ai", "sources": [ { "label": "News", "url": "https://mistral.ai/news", "kind": "html" }, { "label": "Hugging Face org", "url": "https://huggingface.co/api/models?author=mistralai&sort=lastModified&limit=20", "kind": "hf-org" } ], "flagship_hints": [ "Mistral Large \\d", "Mistral Medium \\d", "Magistral", "^Mistral 7B$", "^Mixtral", "^Mistral Large$" ], "name_prefixes": [ { "match": "^(large|medium|small)\\b", "prefix": "Mistral " } ] } ], "benchmarks": [ { "id": "gpqa-diamond", "name": "GPQA Diamond", "short": "GPQA", "description": "198 graduate-level, Google-proof science questions (biology, physics, chemistry) written and validated by PhD experts.", "url": "https://arxiv.org/abs/2311.12022", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 3, "in_index": true, "legacy": false, "preferred_config": "no tools, pass@1 (or lab-reported default)", "human_baseline": 69.7, "human_baseline_note": "PhD-level domain experts scored 69.7% (Rein et al. 2023, Diamond subset).", "introduced": 2023 }, { "id": "swe-bench-verified", "name": "SWE-bench Verified", "short": "SWE-bench", "description": "500 human-validated real GitHub issues; the model must produce a patch that passes the repository tests.", "url": "https://openai.com/index/introducing-swe-bench-verified/", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 3, "in_index": true, "legacy": false, "preferred_config": "lab-reported default scaffold, pass@1 (no parallel test-time compute)", "human_baseline": null, "human_baseline_note": null, "introduced": 2024 }, { "id": "aime", "name": "AIME (American Invitational Mathematics Examination)", "short": "AIME", "description": "30 competition math problems per year with integer answers; labs report the most recent edition (2024 or 2025). Edition recorded in config.", "url": "https://matharena.ai", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 3, "in_index": true, "legacy": false, "preferred_config": "AIME 2025 without tools; fall back to AIME 2024 for models from 2024", "human_baseline": null, "human_baseline_note": "Top ~5% of US high-school contestants qualify; a median qualifier scores roughly 5/15 per exam.", "introduced": 2024 }, { "id": "mmlu-pro", "name": "MMLU-Pro", "short": "MMLU-Pro", "description": "12k harder multiple-choice questions (10 options) across 14 domains; reasoning-heavy successor to MMLU.", "url": "https://arxiv.org/abs/2406.01574", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 3, "in_index": true, "legacy": false, "preferred_config": "lab-reported default (usually CoT, 5-shot or 0-shot)", "human_baseline": null, "human_baseline_note": null, "introduced": 2024 }, { "id": "hle", "name": "Humanity's Last Exam", "short": "HLE", "description": "2,500 expert-written questions across 100+ subjects designed to sit at the frontier of human knowledge.", "url": "https://lastexam.ai", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 4, "in_index": true, "legacy": false, "preferred_config": "no tools, full set (text + multimodal); record 'text-only' or 'with tools' in config when that is all a lab reports", "human_baseline": null, "human_baseline_note": "Each question is solvable by its expert author; no aggregate human score exists.", "introduced": 2025 }, { "id": "arc-agi-2", "name": "ARC-AGI-2", "short": "ARC-AGI-2", "description": "Novel visual abstraction puzzles that humans solve easily and that resist memorisation; measures fluid intelligence.", "url": "https://arcprize.org/arc-agi/2/", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 4, "in_index": true, "legacy": false, "preferred_config": "semi-private evaluation set (ARC Prize verified); public eval if that is all a lab reports", "human_baseline": 100, "human_baseline_note": "Every task was solved by at least two humans in controlled testing; the average test-taker panel scored about 60%.", "introduced": 2025 }, { "id": "terminal-bench-2", "name": "Terminal-Bench 2.0", "short": "Terminal-Bench", "description": "89 hard, verified real-world terminal tasks (compilation, ops, data wrangling) solved end-to-end by an agent in a sandbox.", "url": "https://www.tbench.ai", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 4, "in_index": true, "legacy": false, "preferred_config": "Terminus-2 harness or lab default; version 2.0 only (1.0 is a different benchmark)", "human_baseline": null, "human_baseline_note": null, "introduced": 2025 }, { "id": "tau2-bench", "name": "τ²-bench", "short": "τ²-bench", "description": "Agentic customer-service simulations (retail, airline, telecom) where the model must follow policy while using tools and talking to a simulated user.", "url": "https://github.com/sierra-research/tau2-bench", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 4, "in_index": true, "legacy": false, "preferred_config": "pass^1 average across domains; if only per-domain numbers exist, record the unweighted mean and list domains in note", "human_baseline": null, "human_baseline_note": null, "introduced": 2025 }, { "id": "mmmu", "name": "MMMU", "short": "MMMU", "description": "11.5k college-level multimodal questions spanning 30 subjects; requires reading diagrams, charts and images.", "url": "https://mmmu-benchmark.github.io", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 3, "in_index": true, "legacy": false, "preferred_config": "validation split, original MMMU (record 'MMMU-Pro' in config when that is what a lab reports)", "human_baseline": 88.6, "human_baseline_note": "Human experts scored 88.6% on the validation set (Yue et al. 2023).", "introduced": 2023 }, { "id": "livecodebench", "name": "LiveCodeBench", "short": "LiveCodeBench", "description": "Continuously refreshed competitive-programming problems collected after model training cut-offs to resist contamination.", "url": "https://livecodebench.github.io", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 3, "in_index": true, "legacy": false, "preferred_config": "pass@1; record the problem window/version (e.g. v6) in config", "human_baseline": null, "human_baseline_note": null, "introduced": 2024 }, { "id": "mmlu", "name": "MMLU", "short": "MMLU", "description": "57-subject multiple-choice knowledge test; saturated since 2024 but anchors the 2023–2024 generation.", "url": "https://arxiv.org/abs/2009.03300", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 2, "in_index": true, "legacy": true, "preferred_config": "5-shot (or lab default)", "human_baseline": 89.8, "human_baseline_note": "Estimated expert-level accuracy 89.8% (Hendrycks et al. 2021).", "introduced": 2020 }, { "id": "humaneval", "name": "HumanEval", "short": "HumanEval", "description": "164 hand-written Python programming problems checked by unit tests; saturated, kept as a legacy anchor.", "url": "https://arxiv.org/abs/2107.03374", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 2, "in_index": true, "legacy": true, "preferred_config": "pass@1, 0-shot", "human_baseline": null, "human_baseline_note": null, "introduced": 2021 }, { "id": "math", "name": "MATH (Hendrycks)", "short": "MATH", "description": "12.5k competition mathematics problems; saturated by 2025, kept as a legacy anchor (MATH-500 accepted, noted in config).", "url": "https://arxiv.org/abs/2103.03874", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 2, "in_index": true, "legacy": true, "preferred_config": "full test set or MATH-500 (record which)", "human_baseline": null, "human_baseline_note": "A three-time IMO gold medalist scored 90% on a sample (Hendrycks et al. 2021).", "introduced": 2021 }, { "id": "gsm8k", "name": "GSM8K", "short": "GSM8K", "description": "8.5k grade-school math word problems; saturated since 2024 but links the 2021–2024 generations of every lab.", "url": "https://arxiv.org/abs/2110.14168", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 2, "in_index": true, "legacy": true, "preferred_config": "few-shot chain-of-thought (or lab default)", "human_baseline": null, "human_baseline_note": null, "introduced": 2021 }, { "id": "hellaswag", "name": "HellaSwag", "short": "HellaSwag", "description": "Adversarially filtered commonsense sentence completion; saturated since 2023, anchors the 2019–2023 generations (GPT-3, PaLM, LLaMA).", "url": "https://arxiv.org/abs/1905.07830", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 1, "in_index": true, "legacy": true, "preferred_config": "10-shot (or lab default)", "human_baseline": 95.6, "human_baseline_note": "Human accuracy 95.6% (Zellers et al. 2019).", "introduced": 2019 }, { "id": "arc-challenge", "name": "ARC Challenge", "short": "ARC-C", "description": "Grade-school science questions that retrieval and co-occurrence methods fail (Challenge set); saturated since 2023.", "url": "https://arxiv.org/abs/1803.05457", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 1, "in_index": true, "legacy": true, "preferred_config": "25-shot (or lab default)", "human_baseline": null, "human_baseline_note": null, "introduced": 2018 }, { "id": "winogrande", "name": "WinoGrande", "short": "WinoGrande", "description": "Large-scale Winograd-schema pronoun resolution with adversarial filtering; saturated since 2023.", "url": "https://arxiv.org/abs/1907.10641", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 1, "in_index": true, "legacy": true, "preferred_config": "5-shot (or lab default)", "human_baseline": 94, "human_baseline_note": "Human performance 94.0% (Sakaguchi et al. 2019).", "introduced": 2019 }, { "id": "lambada", "name": "LAMBADA", "short": "LAMBADA", "description": "Last-word prediction on narrative passages that need the whole context; the one benchmark GPT-2, GPT-3, PaLM and GLM-130B all reported.", "url": "https://arxiv.org/abs/1606.06031", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 1, "in_index": true, "legacy": true, "preferred_config": "zero-shot accuracy (or lab default)", "human_baseline": null, "human_baseline_note": null, "introduced": 2016 }, { "id": "lmarena-text", "name": "LMArena (text)", "short": "Arena", "description": "Community Elo from blind pairwise human votes on lmarena.ai (text leaderboard, style control). A rating, not a test — converted to the probability of beating a 1200-rated reference before it enters the fit.", "url": "https://lmarena.ai/leaderboard/text", "unit": "elo", "min": 0, "max": 4000, "higher_is_better": true, "elo_reference": 1200, "weight": 2, "generation": 3, "community": true, "in_index": true, "legacy": false, "preferred_config": "text leaderboard, style control (default view)", "human_baseline": null, "human_baseline_note": null, "introduced": 2023 }, { "id": "swe-bench-pro", "name": "SWE-bench Pro", "short": "SWE-bench Pro", "description": "Harder, contamination-resistant successor to SWE-bench Verified (Scale AI). Recorded, not yet in the index (too few labs report it).", "url": "https://scale.com/leaderboard/swe_bench_pro_public", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 4, "in_index": false, "legacy": false, "preferred_config": "public set", "human_baseline": null, "human_baseline_note": null, "introduced": 2025 }, { "id": "browsecomp", "name": "BrowseComp", "short": "BrowseComp", "description": "1,266 hard-to-find facts that require persistent multi-step web browsing to locate. Recorded, not in the index.", "url": "https://openai.com/index/browsecomp/", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 4, "in_index": false, "legacy": false, "preferred_config": "with browsing tools", "human_baseline": null, "human_baseline_note": "Human trainers solved 29.2% of problems within a 2-hour limit.", "introduced": 2025 }, { "id": "osworld", "name": "OSWorld / OSWorld-Verified", "short": "OSWorld", "description": "369 real computer-use tasks across operating systems and applications executed by an agent. Recorded, not in the index.", "url": "https://os-world.github.io", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 3, "in_index": false, "legacy": false, "preferred_config": "OSWorld-Verified, record step budget in config", "human_baseline": 72.36, "human_baseline_note": "Human success rate 72.36% (Xie et al. 2024).", "introduced": 2024 }, { "id": "frontiermath", "name": "FrontierMath", "short": "FrontierMath", "description": "Unpublished research-level mathematics problems evaluated by Epoch AI. Usually reported by the maintainer, not labs. Recorded, not in the index.", "url": "https://epoch.ai/frontiermath", "unit": "%", "min": 0, "max": 100, "higher_is_better": true, "weight": 1, "generation": 4, "in_index": false, "legacy": false, "preferred_config": "Tiers 1–3 (record tier in config)", "human_baseline": null, "human_baseline_note": null, "introduced": 2024 } ], "releases": [ { "id": "openai-gpt-1", "lab": "openai", "name": "GPT-1", "family": "GPT", "date": "2018-06-11", "date_precision": "day", "status": "released", "announcement": { "url": "https://github.com/openai/finetune-transformer-lm", "title": "finetune-transformer-lm", "quote": "Code and model for the paper \"Improving Language Understanding by Generative Pre-Training\"", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "sources": [ { "url": "https://openai.com/index/language-unsupervised/", "title": "Improving language understanding with unsupervised learning", "quote": "Our system works in two stages; first we train a transformer model on a very large amount of data in an unsupervised manner", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } ], "scores": [], "notes": "Open-weights research release (paper 'Improving Language Understanding by Generative Pre-Training', Radford et al.) with code and model published on GitHub; no self-serve product or API. Neither the GitHub README nor the OpenAI blog post states a publication date in their own extracted text; the exact date is corroborated by the GitHub repository's earliest commits, all timestamped 2018-06-11T06:04–07:00 UTC per the GitHub API. No index-basket benchmark scores were officially reported for GPT-1 (its evaluation suite predates MMLU/HellaSwag/GSM8K-era benchmarks); recorded with zero scores so it still appears as a timeline tick." }, { "id": "openai-gpt-2", "lab": "openai", "name": "GPT-2", "family": "GPT", "date": "2019-11-05", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/gpt-2-1-5b-release/", "title": "GPT-2: 1.5B release", "quote": "As the final model release of GPT‑2's staged release, we're releasing the largest version (1.5B parameters) of GPT‑2 along with code and model weights to facilitate detection of outputs of GPT‑2 models.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "sources": [ { "url": "https://openai.com/index/better-language-models/", "title": "Better language models and their implications", "quote": "LAMBADA accuracy (+)63.24%59.23%95%+", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } ], "scores": [ { "benchmark": "lambada", "value": 63.24, "config": "zero-shot, full 1.5B model", "note": "Value is for the full 1.5B model as reported in the original Feb 2019 announcement post and its companion technical paper, prior to the staged public release of the weights themselves on Nov 5, 2019.", "reported_by": "official", "source": { "url": "https://openai.com/index/better-language-models/", "title": "Better language models and their implications", "quote": "LAMBADA accuracy (+)63.24%59.23%95%+", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } } ], "notes": "Flagship = the 1.5B-parameter model, released in full (weights + code) on November 5, 2019 as the final stage of GPT‑2's staged release strategy (an initial 117M 'small' version was released in Feb 2019, citing misuse concerns; 345M followed in May 2019, 774M in Aug 2019). Announcement quote and date are from the Nov 5, 2019 final-release post; the LAMBADA score was reported earlier, in the original Feb 14, 2019 post and paper, for the full model prior to its staged public release." }, { "id": "openai-gpt-3", "lab": "openai", "name": "GPT-3", "family": "GPT", "date": "2020-06-11", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/openai-api/", "title": "OpenAI API", "quote": "We’re releasing an API for accessing new AI models developed by OpenAI.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "sources": [ { "url": "https://ar5iv.labs.arxiv.org/html/2005.14165", "title": "Language Models are Few-Shot Learners", "retrieved_at": "2026-09-07T09:30:00Z" }, { "url": "https://ar5iv.labs.arxiv.org/html/2107.03374", "title": "Evaluating Large Language Models Trained on Code", "retrieved_at": "2026-09-07T09:30:00Z" } ], "scores": [ { "benchmark": "lambada", "value": 76, "config": "zero-shot, 175B", "note": "Paper text rounds to 76%; Table 3.2 gives a more precise 76.2% but that table did not render as extractable text in this fetch.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2005.14165", "title": "Language Models are Few-Shot Learners", "quote": "in a zero-shot setting GPT-3 achieves 76% on LAMBADA, a gain of 8% over the previous state of the art.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hellaswag", "value": 79.3, "config": "few-shot, 175B", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2005.14165", "title": "Language Models are Few-Shot Learners", "quote": "GPT-3 achieves 78.1% accuracy in the one-shot setting and 79.3% accuracy in the few-shot setting", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "arc-challenge", "value": 51.5, "config": "few-shot, 175B, Challenge set", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2005.14165", "title": "Language Models are Few-Shot Learners", "quote": "GPT-3 achieves 51.4% accuracy in the zero-shot setting, 53.2% in the one-shot setting, and 51.5% in the few-shot setting.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "winogrande", "value": 77.7, "config": "few-shot, 175B", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2005.14165", "title": "Language Models are Few-Shot Learners", "quote": "GPT-3 achieves 70.2% in the zero-shot setting, 73.2% in the one-shot setting, and 77.7% in the few-shot setting.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "humaneval", "value": 0, "config": "0-shot, 175B", "note": "GPT-3's own paper does not evaluate code generation; this baseline figure is from OpenAI's companion Codex paper, which re-ran GPT-3 on HumanEval as a baseline.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2107.03374", "title": "Evaluating Large Language Models Trained on Code", "quote": "our model solves 28.8% of the problems, while GPT-3 solves 0% and GPT-J solves 11.4%.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmlu", "value": 43.9, "config": "few-shot, 175B (X-Large)", "note": "MMLU postdates the GPT-3 paper; this number is from the MMLU benchmark authors' own paper (Hendrycks et al.), which evaluated GPT-3 via the OpenAI API, not from an OpenAI publication.", "reported_by": "maintainer", "source": { "url": "https://arxiv.org/pdf/2009.03300", "title": "Measuring Massive Multitask Language Understanding", "quote": "the X-Large 175 billion parameter GPT-3 model performs substantially better than random, with an accuracy of 43.9%.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } } ], "notes": "Public API beta launched June 11, 2020 ('launching today in a private beta rather than general availability' per the announcement); general availability followed later (Nov 18, 2021). GSM8K is not recorded: OpenAI's own GSM8K paper (arXiv:2110.14168) only reports fine-tuned/verifier results for GPT-3 175B, not a plain few-shot number. MATH is not recorded: the Hendrycks MATH paper reports only fine-tuned MATH accuracy for GPT-3 (Curie/Davinci), not a clean few-shot baseline comparable to this project's convention." }, { "id": "zhipu-glm-130b", "lab": "zhipu", "name": "GLM-130B", "family": "GLM", "date": "2022-08-01", "date_precision": "month", "status": "released", "announcement": { "url": "https://raw.githubusercontent.com/THUDM/GLM-130B/main/README.md", "title": "THUDM/GLM-130B GitHub README", "quote": "GLM-130B is an open bilingual (English & Chinese) bidirectional dense model with 130 billion parameters, pre-trained using the algorithm of General Language Model (GLM).", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Weights were released for research access via an application form in Aug 2022 (commonly cited as Aug 4, 2022); this exact day is not confirmed by the GitHub README's own changelog (which starts at 2022.08.24, a later quantization update), so recorded at month precision. Training itself ran May 6 - Jul 3, 2022 per the paper (arXiv 2210.02414)." }, "scores": [ { "benchmark": "lambada", "value": 80.2, "config": "zero-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2210.02414", "title": "GLM-130B: An Open Bilingual Pre-trained Model", "quote": "GLM-130B offers a record-high accuracy of 80.2% on zero-shot LAMBADA by outperforming both GPT-3 and PaLM 540B in Figure 2.", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "A companion table (Table 2) gives the more precise FP16 value 80.21, rounded to 80.2 in this prose sentence." } }, { "benchmark": "mmlu", "value": 44.8, "config": "5-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2210.02414", "title": "GLM-130B: An Open Bilingual Pre-trained Model", "quote": "It continues moving up as the training proceeds, achieving an accuracy of 44.8 when the training has to end (i.e., viewing 400B tokens in total).", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Sentence follows 'GLM-130B's few-shot (5-shot) performance on MMLU approaches GPT-3 (43.9)...'; a companion table (Table 2) gives the more precise FP16 value 44.75, rounded to 44.8 in this prose sentence." } } ], "notes": "GLM-130B was Zhipu/Tsinghua's flagship open-weights model before ChatGLM-6B; MMLU and LAMBADA are the only two index/legacy benchmarks the paper reports in a form directly comparable to this dataset's basket (most other reported results are on Chinese-specific or held-out benchmarks)." }, { "id": "openai-gpt-3.5", "lab": "openai", "name": "GPT-3.5 (ChatGPT)", "family": "GPT", "date": "2022-11-30", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/chatgpt/", "title": "Introducing ChatGPT", "quote": "We are excited to introduce ChatGPT to get users’ feedback and learn about its strengths and weaknesses. During the research preview, usage of ChatGPT is free.", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "sources": [ { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "retrieved_at": "2026-09-07T09:30:00Z" } ], "scores": [ { "benchmark": "mmlu", "value": 70, "config": "5-shot, GPT-4 Technical Report Table 2", "note": "Table 2, columns GPT-4/GPT-3.5/best-LM-SOTA/best-overall-SOTA; 70.0% is GPT-3.5's value.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "86.4%70.0%70.7%75.2%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hellaswag", "value": 85.5, "config": "10-shot, GPT-4 Technical Report Table 2", "note": "Table 2, columns GPT-4/GPT-3.5/best-LM-SOTA/best-overall-SOTA; 85.5% is GPT-3.5's value.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "95.3%85.5%84.2%85.6", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "arc-challenge", "value": 85.2, "config": "25-shot, GPT-4 Technical Report Table 2", "note": "Table 2, columns GPT-4/GPT-3.5/best-LM-SOTA/best-overall-SOTA; 85.2% is GPT-3.5's value.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "96.3%85.2%85.2%86.5%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "winogrande", "value": 81.6, "config": "5-shot, GPT-4 Technical Report Table 2", "note": "Table 2, columns GPT-4/GPT-3.5/best-LM-SOTA/best-overall-SOTA; 81.6% is GPT-3.5's value.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "87.5%81.6%85.1%85.1%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "humaneval", "value": 48.1, "config": "0-shot, GPT-4 Technical Report Table 2", "note": "Table 2, columns GPT-4/GPT-3.5/best-LM-SOTA/best-overall-SOTA; 48.1% is GPT-3.5's value.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "67.0%48.1%26.2%65.8%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gsm8k", "value": 57.1, "config": "5-shot, GPT-4 Technical Report Table 2", "note": "Table 2, columns GPT-4/GPT-3.5/best-LM-SOTA/best-overall-SOTA; 57.1% is GPT-3.5's value (GPT-4's own value carries a footnote asterisk not reproduced here).", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "57.1%58.8%87.3%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } } ], "notes": "ChatGPT is a research preview (free during initial rollout) fine-tuned from a model in the GPT-3.5 series ('ChatGPT is fine-tuned from a model in the GPT‑3.5 series, which finished training in early 2022', per the launch post). Launch date (November 30, 2022) is not stated in the post's own extracted text but is one of the most widely and consistently reported dates in contemporaneous tech press. No benchmark table was published alongside the ChatGPT launch post itself; scores here are OpenAI's own 'GPT-3.5' comparison column from Table 2 of the GPT-4 Technical Report (arXiv:2303.08774), the same table used for this file's GPT-4 legacy scores." }, { "id": "meta-llama-1-65b", "lab": "meta", "name": "LLaMA 65B", "family": "Llama", "date": "2023-02-24", "date_precision": "day", "status": "released", "announcement": { "url": "https://ai.meta.com/blog/large-language-model-llama-meta-ai/", "title": "Introducing LLaMA: A foundational, 65-billion-parameter language model", "quote": "As part of Meta's commitment to open science, today we are publicly releasing LLaMA (Large Language Model Meta AI), a state-of-the-art foundational large language model designed to help researchers advance their work in this subfield of AI.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "mmlu", "value": 63.4, "config": "reported in the Llama 2 model card's Llama 1/Llama 2 comparison table", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama/blob/main/MODEL_CARD.md", "title": "Llama 2 Model Card (Meta, comparison table includes Llama 1 65B)", "quote": "|Llama 1|65B|30.7|70.7|60.5|68.6|30.8|63.4|43.5|47.6|", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "hellaswag", "value": 84.2, "config": "0-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2302.13971", "title": "LLaMA: Open and Efficient Foundation Language Models (arXiv:2302.13971, Table 3)", "quote": "65B 85.3 82.8 52.3 84.2 77.0 78.9 56.0 60.2", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Table 3 row for the 65B model. Columns: BoolQ, PIQA, SIQA, HellaSwag, WinoGrande, ARC-e, ARC-c, OBQA." } }, { "benchmark": "winogrande", "value": 77, "config": "0-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2302.13971", "title": "LLaMA: Open and Efficient Foundation Language Models (arXiv:2302.13971, Table 3)", "quote": "65B 85.3 82.8 52.3 84.2 77.0 78.9 56.0 60.2", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Table 3 row for the 65B model. Columns: BoolQ, PIQA, SIQA, HellaSwag, WinoGrande, ARC-e, ARC-c, OBQA." } }, { "benchmark": "arc-challenge", "value": 56, "config": "0-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2302.13971", "title": "LLaMA: Open and Efficient Foundation Language Models (arXiv:2302.13971, Table 3)", "quote": "65B 85.3 82.8 52.3 84.2 77.0 78.9 56.0 60.2", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Table 3 row for the 65B model. Columns: BoolQ, PIQA, SIQA, HellaSwag, WinoGrande, ARC-e, ARC-c (this is ARC-Challenge), OBQA." } }, { "benchmark": "gsm8k", "value": 50.9, "config": "pass@1, no maj1@k", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2302.13971", "title": "LLaMA: Open and Efficient Foundation Language Models (arXiv:2302.13971, Table 7)", "quote": "65B 10.6 20.5 50.9 69.7", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Table 7 row for the 65B model. Columns: MATH, MATH+maj1@k, GSM8K, GSM8K+maj1@k. Value used is the no-majority-voting GSM8K column (50.9); cross-checked against the same figure reproduced in the Llama 2 paper's Table 25 (arXiv:2307.09288)." } }, { "benchmark": "humaneval", "value": 23.7, "config": "0-shot, pass@1 (temperature 0.1)", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2302.13971", "title": "LLaMA: Open and Efficient Foundation Language Models (arXiv:2302.13971, Table 8)", "quote": "65B 23.7 79.3 37.7 76.8", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Table 8 row for the 65B model. Columns: HumanEval pass@1, HumanEval pass@100, MBPP pass@1, MBPP pass@80." } } ], "notes": "Research-only release under a noncommercial license (application required); not a product/API launch. Included as the origin of the Llama family. Sizes 7B/13B/33B/65B were released together; 65B was the largest/flagship." }, { "id": "openai-gpt-4", "lab": "openai", "name": "GPT-4", "family": "GPT", "date": "2023-03-14", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/gpt-4-research/", "title": "GPT-4", "quote": "We are releasing GPT‑4's text input capability via ChatGPT and the API (with a waitlist).", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "mmlu", "value": 86.4, "config": "5-shot, GPT-4 Technical Report Table 2", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2303.08774", "title": "GPT-4 Technical Report", "quote": "86.4%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "humaneval", "value": 67, "config": "0-shot, GPT-4 Technical Report Table 2", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2303.08774", "title": "GPT-4 Technical Report", "quote": "67.0%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gsm8k", "value": 92, "config": "5-shot chain-of-thought, GPT-4 Technical Report Table 2", "note": "OpenAI includes part of the GSM8K training set in GPT-4's pre-training mix (see the report's Appendix E), so this figure sits between true few-shot transfer and full benchmark-specific tuning, per the report's own caveat.", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "92.0%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hellaswag", "value": 95.3, "config": "10-shot, GPT-4 Technical Report Table 2", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "95.3%85.5%84.2%85.6", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "arc-challenge", "value": 96.3, "config": "25-shot, GPT-4 Technical Report Table 2", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "96.3%85.2%85.2%86.5%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "winogrande", "value": 87.5, "config": "5-shot, GPT-4 Technical Report Table 2", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2303.08774", "title": "GPT-4 Technical Report", "quote": "87.5%81.6%85.1%85.1%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } } ], "notes": "Launch date (March 14, 2023) is not stated in the announcement post's body text; corroborated by the pinned API model version string 'gpt-4-0314' quoted on the same page and by widely reported press coverage of the launch. MMLU/HumanEval figures are from Table 2 of the GPT-4 Technical Report (arXiv:2303.08774), fetched via the ar5iv HTML mirror since the announcement page renders its benchmark comparison as an image chart with no extractable text. GSM8K/HellaSwag/ARC-Challenge/WinoGrande legacy scores added from the same Table 2." }, { "id": "anthropic-claude-1.3", "lab": "anthropic", "name": "Claude 1.3", "family": "Claude", "date": "2023-03-14", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/introducing-claude", "title": "Introducing Claude", "quote": "we’ve been able to carefully test out our systems in the wild, and are ready to offer Claude more broadly", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "retrieved_at": "2026-09-07T09:30:00Z", "note": "Claude 1.3 has no model card of its own; its benchmark numbers appear as a comparison column in the later Claude 2 model card." } ], "scores": [ { "benchmark": "humaneval", "value": 56, "config": "0-shot, Codex P@1", "note": "Model card table (Claude Instant / Claude 1.3 / Claude 2 columns); 56.0% is Claude 1.3's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "quote": "Codex P@1 (0-shot) 52.8% 56.0% 71.2%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "gsm8k", "value": 85.2, "config": "0-shot chain-of-thought", "note": "Model card table (Claude Instant / Claude 1.3 / Claude 2 columns); 85.2% is Claude 1.3's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "quote": "GSM8k (0-shot CoT) 80.9% 85.2% 88.0%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "mmlu", "value": 77, "config": "5-shot chain-of-thought", "note": "Model card table (Claude Instant / Claude 1.3 / Claude 2 columns); 77.0% is Claude 1.3's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "quote": "MMLU(5-shotCoT) 73.4% 77.0% 78.5%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "arc-challenge", "value": 90, "config": "5-shot", "note": "Model card table (Claude Instant / Claude 1.3 / Claude 2 columns); 90.0% is Claude 1.3's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "quote": "ARC-Challenge (5-shot) 85.7% 90.0% 91.0%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } } ], "notes": "First public, self-serve release of Claude ('Claude' and 'Claude Instant', accessible via chat interface and API in closed/early access); Claude 1.3 is the specific dated snapshot Anthropic later compared against Claude 2 in the Claude 2 model card. Launch date (March 14, 2023) is not stated in the announcement post's own extracted text; well documented by contemporaneous press coverage. No benchmark table accompanied the original March 2023 announcement itself; scores here are sourced from the Claude 2 model card's cross-model comparison table (Claude Instant / Claude 1.3 / Claude 2 columns)." }, { "id": "zhipu-chatglm-6b", "lab": "zhipu", "name": "ChatGLM-6B", "family": "GLM", "date": "2023-03-14", "date_precision": "day", "status": "released", "announcement": { "url": "https://raw.githubusercontent.com/THUDM/GLM-130B/main/README.md", "title": "THUDM/GLM-130B GitHub README (changelog)", "quote": "[2023.03.14] We are happy to introduce ChatGLM, a bilingual dialogue language model based on GLM-130B, and its open-sourced version ChatGLM-6B which can be run under only 6GB GPU memory!", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 995, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "396 396 399 chatglm-6b Non-commercial 995 ±13 4,914", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "ChatGLM-6B (6.2B parameters) was not Zhipu's largest model at the time — GLM-130B (130B) predates it by seven months — but GLM-130B was a research-access checkpoint requiring an 8-GPU server, with no public chat product. ChatGLM-6B was Zhipu's first publicly accessible flagship chat product (ran locally on a single consumer GPU, and via a public web demo). Recorded here as a timeline tick with no basket scores: the model's GitHub README does not report legacy-basket benchmark numbers for this checkpoint (only for its June 2023 successor, ChatGLM2-6B)." }, { "id": "google-palm-2", "lab": "google", "name": "PaLM 2", "family": "PaLM", "date": "2023-05-10", "date_precision": "day", "status": "released", "announcement": { "url": "https://blog.google/technology/ai/google-palm-2-ai-large-language-model/", "title": "Introducing PaLM 2", "quote": "today we’re introducing PaLM 2, our next generation language model.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "scores": [ { "benchmark": "mmlu", "value": 78.4, "config": "5-shot, PaLM 2-L (Unicorn)", "note": "Not reported in Google's own PaLM 2 technical report; sourced instead from the official Gemini 1.0 technical report's cross-model comparison table, which lists PaLM 2-L's own published MMLU number.", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf", "title": "Gemini: A Family of Highly Capable Multimodal Models", "quote": "78.4% 5-shot", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "math", "value": 34.3, "config": "4-shot, chain-of-thought", "note": "With self-consistency (64 sample paths) PaLM 2 scores 48.8% on the same table; 34.3% is the plain 4-shot CoT figure.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2305.10403", "title": "PaLM 2 Technical Report", "quote": "MATH 50.3a 8.8 33.6 / 50.3 42.5 34.3 / 48.8 33.2 / 45.2", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "lambada", "value": 86.9, "config": "1-shot, PaLM 2-L (Unicorn)", "note": "Table 2 (1-shot English QA/classification tasks), columns PaLM / PaLM 2-S / PaLM 2-M / PaLM 2-L; 86.9 is PaLM 2-L's value.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2305.10403", "title": "PaLM 2 Technical Report", "quote": "LAMBADA 81.8 80.7 83.7 86.9", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "hellaswag", "value": 86.8, "config": "1-shot, PaLM 2-L (Unicorn)", "note": "Table 2, columns PaLM / PaLM 2-S / PaLM 2-M / PaLM 2-L; 86.8 is PaLM 2-L's value.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2305.10403", "title": "PaLM 2 Technical Report", "quote": "HellaSwag 83.6 82.0 84.0 86.8", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "arc-challenge", "value": 69.2, "config": "1-shot, PaLM 2-L (Unicorn)", "note": "Table 2, columns PaLM / PaLM 2-S / PaLM 2-M / PaLM 2-L; 69.2 is PaLM 2-L's value.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2305.10403", "title": "PaLM 2 Technical Report", "quote": "ARC-C 60.1 59.6 64.9 69.2", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "winogrande", "value": 83, "config": "1-shot, PaLM 2-L (Unicorn)", "note": "Table 2, columns PaLM / PaLM 2-S / PaLM 2-M / PaLM 2-L; 83.0 is PaLM 2-L's value.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2305.10403", "title": "PaLM 2 Technical Report", "quote": "WSC 86.3 84.6 88.1 86.9 WinoGrande 83.7 77.9 79.2 83.0", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "gsm8k", "value": 80.7, "config": "8-shot chain-of-thought, PaLM 2-L (Unicorn)", "note": "With self-consistency (40 sample paths) PaLM 2 scores 91.0% on the same table; 80.7% is the plain 8-shot CoT figure, matching this file's existing convention for the MATH score above.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2305.10403", "title": "PaLM 2 Technical Report", "quote": "GSM8K 92.0 b 56.5 / 74.4 58.8 / 78.5 92.0 80.7 / 91.0 84.7 / 92.2", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "lmarena-text", "value": 1138, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "369 355 378 palm-2 Google · Proprietary 1138 ±9 8,554", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "First public availability May 10, 2023 via the PaLM API (open sign-up), Vertex AI and Bard; came in four sizes (Gecko/Otter/Bison/Unicorn). GPQA and MMMU did not exist yet at launch (both introduced later in 2023), and the technical report does not report a HumanEval number for the largest (Unicorn/L) size, only for the small coding-specialized PaLM 2-S*. LAMBADA/HellaSwag/ARC-Challenge/WinoGrande/GSM8K legacy scores added from the technical report's Table 2 (1-shot) and Table 7 (math)." }, { "id": "anthropic-claude-2", "lab": "anthropic", "name": "Claude 2", "family": "Claude", "date": "2023-07-11", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-2", "title": "Claude 2", "quote": "anyone in the US and UK can start using our beta chat experience today.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "retrieved_at": "2026-09-07T05:00:00Z" } ], "scores": [ { "benchmark": "mmlu", "value": 78.5, "config": "5-shot CoT", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "quote": "MMLU(5-shotCoT) 73.4% 77.0% 78.5%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Model card table (Claude Instant / Claude 1.3 / Claude 2 columns); text extraction lost inter-word spaces. 78.5% is Claude 2's value." } }, { "benchmark": "humaneval", "value": 71.2, "config": "0-shot, Codex P@1", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-2", "quote": "Claude 2 scored a 71.2% up from 56.0% on the Codex HumanEval, a Python coding test.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "gsm8k", "value": 88, "config": "0-shot chain-of-thought", "note": "Model card table (Claude Instant / Claude 1.3 / Claude 2 columns); 88.0% is Claude 2's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "quote": "GSM8k (0-shot CoT) 80.9% 85.2% 88.0%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "arc-challenge", "value": 91, "config": "5-shot", "note": "Model card table (Claude Instant / Claude 1.3 / Claude 2 columns); 91.0% is Claude 2's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "title": "Model Card: Claude 2", "quote": "ARC-Challenge (5-shot) 85.7% 90.0% 91.0%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } } ], "notes": "First Claude model with public self-serve access (claude.ai beta, US/UK). GPQA and MMMU did not exist as public benchmarks yet at launch. GSM8K/ARC-Challenge legacy scores added from the Claude 2 model card's comparison table." }, { "id": "meta-llama-2-70b", "lab": "meta", "name": "Llama 2 70B", "family": "Llama", "date": "2023-07-18", "date_precision": "day", "status": "released", "announcement": { "url": "https://ai.meta.com/blog/llama-2/", "title": "Meta and Microsoft Introduce the Next Generation of Llama", "quote": "We're now ready to open source the next version of Llama 2 and are making it available free of charge for research and commercial use.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "mmlu", "value": 68.9, "config": "internal eval harness, as reported in the Llama 2 model card", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama/blob/main/MODEL_CARD.md", "title": "Llama 2 Model Card (Meta)", "quote": "|Llama 2|70B|**37.5**|**71.9**|**63.6**|**69.4**|**35.2**|**68.9**|**51.2**|**54.2**|", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "hellaswag", "value": 85.3, "config": "0-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2307.09288", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models (arXiv:2307.09288, Table 20)", "quote": "70B 85.0 82.8 50.7 85.3 80.2 80.2 57.4 60.2 78.5 68.9", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Appendix Table 20 row for the pretrained Llama 2 70B model. Columns: BoolQ, PIQA, SIQA, HellaSwag, WinoGrande, ARC-e, ARC-c, OBQA, CSQA, MMLU. The MMLU value here (68.9) matches the score already recorded from the model card." } }, { "benchmark": "winogrande", "value": 80.2, "config": "0-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2307.09288", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models (arXiv:2307.09288, Table 20)", "quote": "70B 85.0 82.8 50.7 85.3 80.2 80.2 57.4 60.2 78.5 68.9", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Appendix Table 20 row for the pretrained Llama 2 70B model. Columns: BoolQ, PIQA, SIQA, HellaSwag, WinoGrande, ARC-e, ARC-c, OBQA, CSQA, MMLU." } }, { "benchmark": "arc-challenge", "value": 57.4, "config": "0-shot", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2307.09288", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models (arXiv:2307.09288, Table 20)", "quote": "70B 85.0 82.8 50.7 85.3 80.2 80.2 57.4 60.2 78.5 68.9", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Appendix Table 20 row for the pretrained Llama 2 70B model. Columns: BoolQ, PIQA, SIQA, HellaSwag, WinoGrande, ARC-e, ARC-c (this is ARC-Challenge), OBQA, CSQA, MMLU." } }, { "benchmark": "gsm8k", "value": 56.8, "config": "8-shot, maj1@1", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2307.09288", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models (arXiv:2307.09288, Table 25)", "quote": "70B 56.8 13.5", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Table 25 row for the pretrained Llama 2 70B model (GSM8K, MATH), maj1@1. Same GSM8K figure (56.8) also appears in the main-text Table 4 closed-source comparison." } }, { "benchmark": "math", "value": 13.5, "config": "4-shot, maj1@1", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2307.09288", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models (arXiv:2307.09288, Table 25)", "quote": "70B 56.8 13.5", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Table 25 row for the pretrained Llama 2 70B model (GSM8K, MATH), maj1@1." } }, { "benchmark": "humaneval", "value": 29.9, "config": "0-shot, pass@1 (temperature 0.1)", "reported_by": "official", "source": { "url": "https://ar5iv.labs.arxiv.org/html/2307.09288", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models (arXiv:2307.09288, Table 21)", "quote": "70B 29.9 89.0 45.0 81.4", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Appendix Table 21 row for the pretrained Llama 2 70B model. Columns: HumanEval pass@1, HumanEval pass@100, MBPP pass@1, MBPP pass@80." } }, { "benchmark": "lmarena-text", "value": 1170, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "351 339 358 llama-2-70b-chat Meta · Llama 2 Community 1170 ±5 38,492", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "alibaba-qwen-7b", "lab": "alibaba", "name": "Qwen-7B", "family": "Qwen", "date": "2023-08-03", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen/", "title": "Introducing Qwen", "quote": "we now release Qwen-1.8B, Qwen-7B, Qwen-14B, and Qwen-72B.", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "This retrospective post (published Jan 2024) is the clearest lab statement of the release; its own table lists Qwen-7B's release date as 23.08.03, and the post states it was '4 months after our first release of Qwen-7B, which is the starting point of our opensource journey of large language models (LLM)'." }, "scores": [ { "benchmark": "mmlu", "value": 56.7, "config": "base model (original Aug 2023 release), 5-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-7B/raw/main/README.md", "title": "Qwen/Qwen-7B model card", "quote": "| Qwen-7B (original) | 56.7 | 59.6 | 51.6 | - | 24.4 | 31.2 | 40.6 | 58.8 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Model card table distinguishes the original Aug 2023 weights ('Qwen-7B (original)') from the later-updated v1.1 weights ('Qwen-7B'); this row is the original release. Table columns: MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, CMMLU." } }, { "benchmark": "gsm8k", "value": 51.6, "config": "base model (original Aug 2023 release), 8-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-7B/raw/main/README.md", "title": "Qwen/Qwen-7B model card", "quote": "| Qwen-7B (original) | 56.7 | 59.6 | 51.6 | - | 24.4 | 31.2 | 40.6 | 58.8 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, CMMLU." } }, { "benchmark": "humaneval", "value": 24.4, "config": "base model (original Aug 2023 release), 0-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-7B/raw/main/README.md", "title": "Qwen/Qwen-7B model card", "quote": "| Qwen-7B (original) | 56.7 | 59.6 | 51.6 | - | 24.4 | 31.2 | 40.6 | 58.8 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, CMMLU." } } ], "notes": "First Qwen release and, per the lab's own retrospective post, 'the starting point of our opensource journey' — flagship by default since no larger Qwen model existed yet. MATH was not reported for the original release ('-' in the model card); HellaSwag/ARC-Challenge were not found in the official README for this checkpoint (a HellaSwag figure of 75.1 appears in the Qwen Technical Report, arXiv 2309.16609, but that page could not be reliably fetched for a verifiable quote)." }, { "id": "alibaba-qwen-14b", "lab": "alibaba", "name": "Qwen-14B", "family": "Qwen", "date": "2023-09-25", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen/", "title": "Introducing Qwen", "quote": "we now release Qwen-1.8B, Qwen-7B, Qwen-14B, and Qwen-72B.", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "This retrospective post (published Jan 2024) is the clearest lab statement of the release; its own table lists Qwen-14B's release date as 23.09.25." }, "scores": [ { "benchmark": "mmlu", "value": 66.3, "config": "base model, 5-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-14B/raw/main/README.md", "title": "Qwen/Qwen-14B model card", "quote": "| **Qwen-14B** | **66.3** | **72.1** | **61.3** | **24.8** | **32.3** | **40.8** | **53.4** | **71.0** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, CMMLU." } }, { "benchmark": "gsm8k", "value": 61.3, "config": "base model, 8-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-14B/raw/main/README.md", "title": "Qwen/Qwen-14B model card", "quote": "| **Qwen-14B** | **66.3** | **72.1** | **61.3** | **24.8** | **32.3** | **40.8** | **53.4** | **71.0** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, CMMLU." } }, { "benchmark": "math", "value": 24.8, "config": "base model, 4-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-14B/raw/main/README.md", "title": "Qwen/Qwen-14B model card", "quote": "| **Qwen-14B** | **66.3** | **72.1** | **61.3** | **24.8** | **32.3** | **40.8** | **53.4** | **71.0** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, CMMLU." } }, { "benchmark": "humaneval", "value": 32.3, "config": "base model, 0-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-14B/raw/main/README.md", "title": "Qwen/Qwen-14B model card", "quote": "| **Qwen-14B** | **66.3** | **72.1** | **61.3** | **24.8** | **32.3** | **40.8** | **53.4** | **71.0** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, CMMLU." } }, { "benchmark": "lmarena-text", "value": 1139, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "368 355 379 qwen-14b-chat Alibaba · Qianwen LICENSE 1139 ±11 4,964", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Qwen-14B was Alibaba's largest/flagship Qwen model at its 2023-09-25 launch — Qwen-72B (and Qwen-1.8B) did not ship until 2023-11-30. HellaSwag/ARC-Challenge were not found in the official README for this checkpoint (a HellaSwag figure of 80.2 appears in the Qwen Technical Report, arXiv 2309.16609, but that page could not be reliably fetched for a verifiable quote)." }, { "id": "mistral-mistral-7b", "lab": "mistral", "name": "Mistral 7B", "family": "Mistral 7B", "date": "2023-09-27", "date_precision": "day", "status": "released", "announcement": { "url": "https://mistral.ai/news/announcing-mistral-7b/", "title": "Mistral 7B", "quote": "Mistral AI team is proud to release Mistral 7B, the most powerful language model for its size to date.", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" }, "sources": [ { "url": "https://ar5iv.labs.arxiv.org/html/2310.06825", "title": "Mistral 7B (arXiv:2310.06825)", "retrieved_at": "2026-09-07T10:00:00Z", "note": "The Mistral 7B paper reports all benchmark comparisons only as bar-chart figures (Figures 4/5), with no extractable numeric text table for individual benchmarks." } ], "scores": [ { "benchmark": "mmlu", "value": 62.5, "config": "5-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mistral 7B 7B 62.5%81.0%74.2%82.2%80.5%54.9%23.2%62.5%26.2%50.2%12.7%50.0%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Mistral's own Mistral 7B blog post and paper render benchmarks only as images; this is the same Mistral 7B baseline row Mistral AI republished as text in its later Mixtral technical report. Table 2 columns: Model, Active Params, MMLU, HellaSwag, WinoGrande, PIQA, ARC-e, ARC-c, NQ, TriviaQA, HumanEval, MBPP, MATH, GSM8K (rendered without separators between cells in this fetch)." } }, { "benchmark": "hellaswag", "value": 81, "config": "0-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mistral 7B 7B 62.5%81.0%74.2%82.2%80.5%54.9%23.2%62.5%26.2%50.2%12.7%50.0%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "See MMLU score note for table/column details." } }, { "benchmark": "winogrande", "value": 74.2, "config": "0-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mistral 7B 7B 62.5%81.0%74.2%82.2%80.5%54.9%23.2%62.5%26.2%50.2%12.7%50.0%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "See MMLU score note for table/column details." } }, { "benchmark": "arc-challenge", "value": 54.9, "config": "0-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mistral 7B 7B 62.5%81.0%74.2%82.2%80.5%54.9%23.2%62.5%26.2%50.2%12.7%50.0%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Arc-c column (ARC-Challenge). See MMLU score note for table/column details." } }, { "benchmark": "gsm8k", "value": 50, "config": "8-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mistral 7B 7B 62.5%81.0%74.2%82.2%80.5%54.9%23.2%62.5%26.2%50.2%12.7%50.0%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Final (rightmost) column, GSM8K, 8-shot with maj@8 per the paper's stated methodology. See MMLU score note for table/column details." } }, { "benchmark": "math", "value": 12.7, "config": "4-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mistral 7B 7B 62.5%81.0%74.2%82.2%80.5%54.9%23.2%62.5%26.2%50.2%12.7%50.0%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Math column, 4-shot with maj@4 per the paper's stated methodology. See MMLU score note for table/column details." } }, { "benchmark": "humaneval", "value": 26.2, "config": "0-shot, pass@1", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mistral 7B 7B 62.5%81.0%74.2%82.2%80.5%54.9%23.2%62.5%26.2%50.2%12.7%50.0%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "HumanE column. This value (26.2%) matches the same figure Mistral separately published in its later archived docs.mistral.ai benchmarks table. See MMLU score note for table/column details." } }, { "benchmark": "lmarena-text", "value": 1110, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "383 375 385 mistral-7b-instruct Mistral · Apache 2.0 1110 ±9 8,977", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "First Mistral AI model release, 7.3B-parameter dense model under Apache 2.0 (open weights). Benchmark scores are Mistral's own re-evaluation numbers, sourced from the comparison table in the later Mixtral technical report since the original Mistral 7B blog post and paper render all benchmark results as chart images with no extractable numeric text." }, { "id": "moonshot-kimi-chat", "lab": "moonshot", "name": "Kimi Chat", "family": "Kimi", "date": "2023-10-09", "date_precision": "day", "status": "released", "announcement": { "url": "https://news.aibase.com/news/1918", "title": "Moonshot AI Launches Kimi Chat, Supporting Input of 200,000 Chinese Characters", "quote": "AI startup Moonshot AI has announced the launch of its intelligent assistant product, Kimi Chat, which supports input of up to 200,000 Chinese characters and has achieved world-leading levels in the field of long-form text.", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Moonshot AI's original Oct 9, 2023 announcement was distributed via its WeChat official account, which is not independently crawlable; this contemporaneous news report (published the next day) is used as the citable source instead, consistent with the project's convention of citing press coverage when no crawlable lab-hosted post exists." }, "scores": [], "notes": "First public Kimi product (a long-context chat assistant, trial access via moonshot.cn), the origin of the Kimi line; opened to general public Nov 16, 2023. No benchmark numbers were published at launch, so this release carries zero scores and is recorded purely as a timeline marker for when Moonshot AI first shipped a public product." }, { "id": "deepseek-llm-67b", "lab": "deepseek", "name": "DeepSeek LLM 67B", "family": "DeepSeek-V", "date": "2023-11-01", "date_precision": "month", "status": "released", "announcement": { "url": "https://github.com/deepseek-ai/DeepSeek-LLM", "title": "DeepSeek LLM: Let there be answers", "quote": "We release the DeepSeek LLM 7B/67B, including both base and chat models, to the public.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "scores": [ { "benchmark": "mmlu", "value": 71.1, "config": "0-shot, Chat model", "reported_by": "official", "source": { "url": "https://github.com/deepseek-ai/DeepSeek-LLM", "title": "DeepSeek LLM README", "quote": "| DeepSeek LLM 67B Chat | 81.5 | 71.1 | 84.1 | 73.8 | 71.7 | 65.2 | 67.8 | 85.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "humaneval", "value": 73.8, "config": "pass@1, Chat model", "reported_by": "official", "source": { "url": "https://github.com/deepseek-ai/DeepSeek-LLM", "title": "DeepSeek LLM README", "quote": "| DeepSeek LLM 67B Chat | 81.5 | 71.1 | 84.1 | 73.8 | 71.7 | 65.2 | 67.8 | 85.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "math", "value": 32.6, "config": "0-shot, Chat model", "reported_by": "official", "source": { "url": "https://github.com/deepseek-ai/DeepSeek-LLM", "title": "DeepSeek LLM README", "quote": "DeepSeek LLM 67B Chat exhibits outstanding performance in coding (HumanEval Pass@1: 73.78) and mathematics (GSM8K 0-shot: 84.1, Math 0-shot: 32.6).", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gsm8k", "value": 84.1, "config": "0-shot, Chat model", "reported_by": "official", "source": { "url": "https://github.com/deepseek-ai/DeepSeek-LLM", "title": "DeepSeek LLM README", "quote": "| DeepSeek LLM 67B Chat | 81.5 | 71.1 | 84.1 | 73.8 | 71.7 | 65.2 | 67.8 | 85.1 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hellaswag", "value": 84, "config": "0-shot, Base model", "note": "HellaSwag is reported only for the Base model in this table, not the Chat model used for the other legacy scores above.", "reported_by": "official", "source": { "url": "https://github.com/deepseek-ai/DeepSeek-LLM", "title": "DeepSeek LLM README", "quote": "| DeepSeek LLM 67B Base | 84.0 | 78.9 | 71.3 | 63.4 | 42.7 | 68.7 | 66.1 | 70.8 | 87.6 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1184, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "337 330 354 deepseek-llm-67b-chat DeepSeek · DeepSeek License 1184 ±11 4,932", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "DeepSeek's first-generation dense (non-MoE) model, 7B and 67B released together; 67B is the flagship. Exact release day not confirmed from a primary DeepSeek source, so recorded at month precision (widely reported as Nov 29, 2023). No ARC-Challenge or WinoGrande figures were found in the official README or paper for this model." }, { "id": "xai-grok-1", "lab": "xai", "name": "Grok-1", "family": "Grok", "date": "2023-11-03", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok", "title": "Announcing Grok", "quote": "We are offering a limited number of users in the United States to try out our Grok prototype and provide valuable feedback that will help us improve its capabilities before a wider release.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" }, "sources": [ { "url": "https://x.ai/news/grok-os", "title": "Open Release of Grok-1", "quote": "We are releasing the base model weights and network architecture of Grok-1, our large language model.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z", "note": "Mar 17, 2024: the raw 314B-parameter base model checkpoint (pre-training concluded Oct 2023) was separately released as open weights under Apache 2.0 — the only Grok generation to get an open-weights release." } ], "scores": [ { "benchmark": "mmlu", "value": 73, "config": "5-shot", "reported_by": "official", "source": { "url": "https://x.ai/news/grok", "title": "Announcing Grok", "quote": "Grok-1, a state-of-the-art language model that is significantly more powerful, achieving 63.2% on the HumanEval coding task and 73% on MMLU.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "humaneval", "value": 63.2, "config": "0-shot", "reported_by": "official", "source": { "url": "https://x.ai/news/grok", "title": "Announcing Grok", "quote": "Grok-1, a state-of-the-art language model that is significantly more powerful, achieving 63.2% on the HumanEval coding task and 73% on MMLU.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "math", "value": 23.9, "config": "4-shot", "reported_by": "official", "source": { "url": "https://x.ai/news/grok", "title": "Announcing Grok", "quote": "| MATH | 15.7% 4-shot | 13.5% 4-shot | 16.0% 4-shot | 23.5% 4-shot | 23.9% 4-shot | 34.6% 4-shot | - | 42.5% 4-shot |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "gsm8k", "value": 62.9, "config": "8-shot", "reported_by": "official", "source": { "url": "https://x.ai/news/grok", "title": "Announcing Grok", "quote": "| GSM8k | 56.8% 8-shot | 56.8% 8-shot | 62.9% 8-shot | 57.1% 8-shot | 62.9% 8-shot | 80.7% 8-shot | 88.0% 8-shot | 92.0% 8-shot |", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z", "note": "Table columns: Grok-0 (33B), LLaMa 2 70B, Inflection-1, GPT-3.5, Grok-1 (this value, 5th column), PaLM 2, Claude 2, GPT-4." } } ], "notes": "xAI's original Nov 3, 2023 post offered Grok-1 as an early-access beta gated by a US-only waitlist ('a limited number of users'); it is recorded as the release date/source per project convention since it is the first public availability with a source. The model's raw base weights were separately open-sourced on Mar 17, 2024 (see sources[]). GPQA and MMMU are not applicable (both postdate this release)." }, { "id": "openai-gpt-4-turbo", "lab": "openai", "name": "GPT-4 Turbo", "family": "GPT", "date": "2023-11-06", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/new-models-and-developer-products-announced-at-devday/", "title": "New models and developer products announced at DevDay", "quote": "Today we're launching a preview of the next generation of this model, GPT‑4 Turbo.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "mmlu", "value": 86.7, "config": "OpenAI simple-evals harness, gpt-4-turbo-2024-04-09", "reported_by": "official", "source": { "url": "https://github.com/openai/simple-evals", "title": "openai/simple-evals", "quote": "gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gpqa-diamond", "value": 49.3, "config": "OpenAI simple-evals harness, gpt-4-turbo-2024-04-09", "reported_by": "official", "source": { "url": "https://github.com/openai/simple-evals", "title": "openai/simple-evals", "quote": "gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "math", "value": 73.4, "config": "OpenAI simple-evals harness, gpt-4-turbo-2024-04-09", "reported_by": "official", "source": { "url": "https://github.com/openai/simple-evals", "title": "openai/simple-evals", "quote": "gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "humaneval", "value": 88.2, "config": "OpenAI simple-evals harness, gpt-4-turbo-2024-04-09", "reported_by": "official", "source": { "url": "https://github.com/openai/simple-evals", "title": "openai/simple-evals", "quote": "gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2", "retrieved_at": "2026-09-07T09:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1324, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "237 229 254 gpt-4-turbo-2024-04-09 OpenAI · Proprietary 1324 ±4 98,114", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Announced at OpenAI DevDay (November 6, 2023) as gpt-4-1106-preview, immediately available via the API. No separate academic benchmark scores were published at launch; the post focuses on context length (128k), pricing and knowledge cutoff (April 2023) rather than a benchmark table. Scores above are for the later gpt-4-turbo-2024-04-09 snapshot (same 'GPT-4 Turbo' product line, per OpenAI's own naming in the simple-evals table), taken from OpenAI's own open-sourced simple-evals results table on GitHub since no benchmark numbers were published for the original Nov 2023 preview snapshot itself." }, { "id": "anthropic-claude-2.1", "lab": "anthropic", "name": "Claude 2.1", "family": "Claude", "date": "2023-11-21", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-2-1", "title": "Claude 2.1", "quote": "Claude 2.1 is available now in our API, and is also powering our chat interface at claude.ai for both the free and Pro tiers.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "scores": [], "notes": "Release focused on a 200K token context window, reduced hallucination rates, tool use, and system prompts rather than academic benchmarks. The companion model card PDF (ModelCardClaudev2_with_appendix_v1.pdf) is an appendix to the original Claude 2 card and repeats the same Claude 2 MMLU/HumanEval numbers rather than publishing new ones for 2.1, so no distinct score is recorded here." }, { "id": "alibaba-qwen-72b", "lab": "alibaba", "name": "Qwen-72B", "family": "Qwen", "date": "2023-11-30", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen/", "title": "Introducing Qwen", "quote": "we now release Qwen-1.8B, Qwen-7B, Qwen-14B, and Qwen-72B.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "This retrospective post (published Jan 2024) is the clearest lab statement of the release; its own table lists Qwen-72B's release date as 23.11.30." }, "scores": [ { "benchmark": "mmlu", "value": 77.4, "config": "base model, 5-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-72B", "title": "Qwen/Qwen-72B model card", "quote": "| **Qwen-72B** | **66.4** | **77.4** | **83.3** | **78.9** | **35.2** | **35.4** | **52.2** | **67.7** | **62.5** | **87.6** | **83.6** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Avg, MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, AGIEval, GaokaoBench, CMMLU." } }, { "benchmark": "gsm8k", "value": 78.9, "config": "base model, 8-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-72B", "title": "Qwen/Qwen-72B model card", "quote": "| **Qwen-72B** | **66.4** | **77.4** | **83.3** | **78.9** | **35.2** | **35.4** | **52.2** | **67.7** | **62.5** | **87.6** | **83.6** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Avg, MMLU, C-Eval, GSM8K, MATH, HumanEval, MBPP, BBH, AGIEval, GaokaoBench, CMMLU." } }, { "benchmark": "math", "value": 35.2, "config": "base model, 4-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-72B", "title": "Qwen/Qwen-72B model card", "quote": "| **Qwen-72B** | **66.4** | **77.4** | **83.3** | **78.9** | **35.2** | **35.4** | **52.2** | **67.7** | **62.5** | **87.6** | **83.6** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "humaneval", "value": 35.4, "config": "base model, 0-shot", "reported_by": "official", "source": { "url": "https://huggingface.co/Qwen/Qwen-72B", "title": "Qwen/Qwen-72B model card", "quote": "| **Qwen-72B** | **66.4** | **77.4** | **83.3** | **78.9** | **35.2** | **35.4** | **52.2** | **67.7** | **62.5** | **87.6** | **83.6** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } } ] }, { "id": "mistral-mixtral-8x7b", "lab": "mistral", "name": "Mixtral 8x7B", "family": "Mixtral", "date": "2023-12-11", "date_precision": "day", "status": "released", "announcement": { "url": "https://mistral.ai/news/mixtral-of-experts/", "title": "Mixtral of experts", "quote": "Today, the team is proud to release Mixtral 8x7B, a high-quality sparse mixture of experts model (SMoE) with open weights.", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" }, "sources": [ { "url": "https://ar5iv.labs.arxiv.org/html/2401.04088", "title": "Mixtral of Experts (arXiv:2401.04088, ar5iv mirror)", "retrieved_at": "2026-09-07T10:00:00Z", "note": "ar5iv failed to render Table 2 as text for this paper (only a stray figure caption came through); used the arxiv.org native HTML mirror instead, which renders the same table as text." } ], "scores": [ { "benchmark": "mmlu", "value": 70.6, "config": "5-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mixtral 8x7B 13B 70.6%84.4%77.2%83.6%83.1%59.7%30.6%71.5%40.2%60.7%28.4%74.4%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 columns: Model, Active Params, MMLU, HellaSwag, WinoGrande, PIQA, ARC-e, ARC-c, NQ, TriviaQA, HumanEval, MBPP, MATH, GSM8K (rendered without separators between cells in this fetch). The official blog post's own benchmark tables are images only." } }, { "benchmark": "hellaswag", "value": 84.4, "config": "0-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mixtral 8x7B 13B 70.6%84.4%77.2%83.6%83.1%59.7%30.6%71.5%40.2%60.7%28.4%74.4%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "See MMLU score note for table/column details." } }, { "benchmark": "winogrande", "value": 77.2, "config": "0-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mixtral 8x7B 13B 70.6%84.4%77.2%83.6%83.1%59.7%30.6%71.5%40.2%60.7%28.4%74.4%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "See MMLU score note for table/column details." } }, { "benchmark": "arc-challenge", "value": 59.7, "config": "0-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mixtral 8x7B 13B 70.6%84.4%77.2%83.6%83.1%59.7%30.6%71.5%40.2%60.7%28.4%74.4%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Arc-c column (ARC-Challenge). See MMLU score note for table/column details." } }, { "benchmark": "gsm8k", "value": 74.4, "config": "8-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mixtral 8x7B 13B 70.6%84.4%77.2%83.6%83.1%59.7%30.6%71.5%40.2%60.7%28.4%74.4%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Final (rightmost) column, GSM8K, 8-shot with maj@8 per the paper's stated methodology. See MMLU score note for table/column details." } }, { "benchmark": "math", "value": 28.4, "config": "4-shot", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mixtral 8x7B 13B 70.6%84.4%77.2%83.6%83.1%59.7%30.6%71.5%40.2%60.7%28.4%74.4%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Math column, 4-shot with maj@4 per the paper's stated methodology. See MMLU score note for table/column details." } }, { "benchmark": "humaneval", "value": 40.2, "config": "0-shot, pass@1", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2401.04088v1", "title": "Mixtral of Experts (arXiv:2401.04088, Table 2)", "quote": "Mixtral 8x7B 13B 70.6%84.4%77.2%83.6%83.1%59.7%30.6%71.5%40.2%60.7%28.4%74.4%", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "HumanE column. This value (40.2%) matches the same figure Mistral separately published in its archived docs.mistral.ai benchmarks table. See MMLU score note for table/column details." } } ], "notes": "Sparse mixture-of-experts model (46.7B total / 12.9B active parameters), open weights under Apache 2.0. The official blog post's benchmark tables are images only; scores here are sourced from the technical paper's text table (Table 2), fetched via the arxiv.org native HTML rendering since the ar5iv mirror did not render this paper's tables as text." }, { "id": "zhipu-glm-4", "lab": "zhipu", "name": "GLM-4", "family": "GLM", "date": "2024-01-16", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.zhipuai.cn/devday", "title": "智谱AI · 发展历程 (DevDay)", "quote": "发布新一代基座大模型 GLM-4，国内率先对标GPT4", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:36:07Z", "note": "Exact day (Jan 16, 2024) corroborated by contemporaneous press (MarkTechPost, Maginative); the lab's own timeline entry gives only month precision." }, "scores": [ { "benchmark": "mmlu", "value": 81.5, "config": "GLM-4 (0116) API model", "note": "Table 2 columns in order: MMLU, GSM8K, MATH, BBH, GPQA, HumanEval. Row is the closed API model shipped 2024-01-16 (version tag '0116'). PDF source: the ar5iv HTML rendering of this 71-page paper is truncated/unreliable, so this and the sibling scores below cite the PDF instead of the (non-quotable) arXiv abstract page previously used.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2406.12793", "title": "ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools — Table 2", "quote": "GLM-4 (0116) 81.5 87.6 47.9 82.3 35.7 72.0", "retrieved_at": "2026-09-07T05:30:00Z", "verified": false, "verified_at": "2026-09-07T09:36:07Z", "note": "PDF source" } }, { "benchmark": "math", "value": 47.9, "config": "GLM-4 (0116) API model", "note": "Table 2 columns in order: MMLU, GSM8K, MATH, BBH, GPQA, HumanEval. Row is the closed API model shipped 2024-01-16 (version tag '0116').", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2406.12793", "title": "ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools — Table 2", "quote": "GLM-4 (0116) 81.5 87.6 47.9 82.3 35.7 72.0", "retrieved_at": "2026-09-07T05:30:00Z", "verified": false, "verified_at": "2026-09-07T09:36:07Z", "note": "PDF source" } }, { "benchmark": "gpqa-diamond", "value": 35.7, "config": "GLM-4 (0116) API model", "note": "Paper reports this simply as 'GPQA' (Rein et al. 2023); Diamond-subset vs. full-set is not specified. Table 2 columns in order: MMLU, GSM8K, MATH, BBH, GPQA, HumanEval.", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2406.12793", "title": "ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools — Table 2", "quote": "GLM-4 (0116) 81.5 87.6 47.9 82.3 35.7 72.0", "retrieved_at": "2026-09-07T05:30:00Z", "verified": false, "verified_at": "2026-09-07T09:36:07Z", "note": "PDF source" } }, { "benchmark": "humaneval", "value": 72, "config": "GLM-4 (0116) API model, pass@1", "note": "Table 2 columns in order: MMLU, GSM8K, MATH, BBH, GPQA, HumanEval. Row is the closed API model shipped 2024-01-16 (version tag '0116').", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2406.12793", "title": "ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools — Table 2", "quote": "GLM-4 (0116) 81.5 87.6 47.9 82.3 35.7 72.0", "retrieved_at": "2026-09-07T05:30:00Z", "verified": false, "verified_at": "2026-09-07T09:36:07Z", "note": "PDF source" } }, { "benchmark": "gsm8k", "value": 87.6, "config": "GLM-4 (0116) API model", "note": "Table 2 columns in order: MMLU, GSM8K, MATH, BBH, GPQA, HumanEval. Row is the closed API model shipped 2024-01-16 (version tag '0116').", "reported_by": "official", "source": { "url": "https://arxiv.org/pdf/2406.12793", "title": "ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools — Table 2", "quote": "GLM-4 (0116) 81.5 87.6 47.9 82.3 35.7 72.0", "retrieved_at": "2026-09-07T05:30:00Z", "verified": false, "verified_at": "2026-09-07T09:36:07Z", "note": "PDF source" } } ] }, { "id": "google-gemini-1.0-ultra", "lab": "google", "name": "Gemini 1.0 Ultra", "family": "Gemini Ultra", "date": "2024-02-08", "date_precision": "day", "status": "released", "announcement": { "url": "https://blog.google/products/gemini/bard-gemini-advanced-app/", "title": "Bard becomes Gemini: Try Ultra 1.0 and a new mobile app today", "quote": "Today we’re launching Gemini Advanced — a new experience that gives you access to Ultra 1.0, our largest and most capable state-of-the-art AI model.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "sources": [ { "url": "https://blog.google/technology/ai/google-gemini-ai/", "title": "Introducing Gemini: our largest and most capable AI model", "quote": "Our first version, Gemini 1.0, is optimized for different sizes: Ultra, Pro and Nano.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z", "note": "Original Dec 6, 2023 unveiling of the Gemini 1.0 family; Ultra itself was not yet publicly usable at that date (\"Gemini Ultra coming soon\")." } ], "scores": [ { "benchmark": "mmlu", "value": 90, "config": "CoT@32, uncertainty-routed", "note": "Headline number from Google's own announcement. The technical report also lists a plain 5-shot figure of 83.7% for Ultra.", "reported_by": "official", "source": { "url": "https://blog.google/technology/ai/google-gemini-ai/", "title": "Introducing Gemini: our largest and most capable AI model", "quote": "With a score of 90.0%, Gemini Ultra is the first model to outperform human experts on MMLU (massive multitask language understanding)", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "humaneval", "value": 74.4, "config": "0-shot, post-trained", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf", "title": "Gemini: A Family of Highly Capable Multimodal Models", "quote": "instruction-tuned Gemini Ultra correctly implements 74.4% of problems", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "math", "value": 53.2, "config": "4-shot", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf", "title": "Gemini: A Family of Highly Capable Multimodal Models", "quote": "53.2% 4-shot", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "mmmu", "value": 59.4, "config": "validation split, pass@1", "note": "The technical report's abstract separately highlights a 62.4% Maj@32 (self-consistency) result for the same model.", "reported_by": "official", "source": { "url": "https://blog.google/technology/ai/google-gemini-ai/", "title": "Introducing Gemini: our largest and most capable AI model", "quote": "Gemini Ultra also achieves a state-of-the-art score of 59.4% on the new MMMU benchmark, which consists of multimodal tasks spanning different domains requiring deliberate reasoning.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "gsm8k", "value": 94.4, "config": "Maj1@32", "note": "PDF source; r.jina.ai returns an empty render for this large technical-report PDF on repeated attempts, so text was extracted locally with pdftotext instead and cannot yet be auto-verified. Cross-confirmed by the Claude 3 model card's independent comparison table, which also lists Gemini 1.0 Ultra's GSM8K score as 94.4% (Maj1@32).", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf", "title": "Gemini: A Family of Highly Capable Multimodal Models", "quote": "the grade-school math benchmark, GSM8K (Cobbe et al., 2021), we find Gemini Ultra reaches 94.4%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } } ], "notes": "Gemini 1.0 Ultra was unveiled Dec 6, 2023 (blog.google/technology/ai/google-gemini-ai/) but was not itself publicly usable then — only Gemini Pro shipped in Bard/API that day. Ultra first became publicly available Feb 8, 2024 as the model behind the new Gemini Advanced subscription (Google One AI Premium, $19.99/mo); that availability date is recorded here as the release date per project convention. GPQA was not reported for Ultra in either the announcement or the technical report. GSM8K legacy score added from the technical report's prose (Section on math benchmarks); HellaSwag was not added despite being requested because the report's own multi-column comparison table renders too ambiguously via both pdftotext and r.jina.ai to confidently attribute a decontaminated 10-shot figure to Gemini Ultra specifically (the report separately warns that a naive contaminated evaluation reads 96.0%, which this project deliberately does not use)." }, { "id": "mistral-mistral-large", "lab": "mistral", "name": "Mistral Large", "family": "Mistral Large", "date": "2024-02-26", "date_precision": "day", "status": "released", "announcement": { "url": "https://mistral.ai/news/mistral-large/", "title": "Au Large", "quote": "We are releasing Mistral Large, our latest and most advanced language model.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" }, "scores": [ { "benchmark": "mmlu", "value": 81.2, "config": "5-shot", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20240229181230/https://docs.mistral.ai/platform/endpoints/", "title": "Mistral AI docs — Benchmark results (archived, Feb 2024)", "quote": "Mistral Small 72.2% 86.9% 84.7% 86.9% 79.5% 51.7% Mistral Medium 75.3% 88.0% 88% 89.9% 81.1% 47% Mistral Large 81.2% 89.2% 86.7% 94.0% 82.7% 50.6%", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Table columns: MMLU, HellaSwag (10-shot), WinoGrande (5-shot), ARC-Challenge (25-shot), TriviaQA (5-shot), TruthfulQA. Current docs.mistral.ai/platform/endpoints page has been replaced; Wayback Machine snapshot from just after the Mistral Large launch used instead." } }, { "benchmark": "humaneval", "value": 45.1, "config": "pass@1", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20240229181230/https://docs.mistral.ai/platform/endpoints/", "title": "Mistral AI docs — Benchmark results (archived, Feb 2024)", "quote": "Model HumanE pass@1 MBPP pass@1 Mistral 7B 26.2% 50.2% Mistral 8x7B 40.2% 60.7% Mistral Small 44.5% 61.5% Mistral Medium 38.4% 62.3% Mistral Large 45.1% 73.2%", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" } }, { "benchmark": "hellaswag", "value": 89.2, "config": "10-shot", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20240229181230/https://docs.mistral.ai/platform/endpoints/", "title": "Mistral AI docs — Benchmark results (archived, Feb 2024)", "quote": "Mistral Small 72.2% 86.9% 84.7% 86.9% 79.5% 51.7% Mistral Medium 75.3% 88.0% 88% 89.9% 81.1% 47% Mistral Large 81.2% 89.2% 86.7% 94.0% 82.7% 50.6%", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Same row/quote already used for this release's MMLU score. Table columns: MMLU, HellaSwag (10-shot), WinoGrande (5-shot), ARC-Challenge (25-shot), TriviaQA (5-shot), TruthfulQA." } }, { "benchmark": "winogrande", "value": 86.7, "config": "5-shot", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20240229181230/https://docs.mistral.ai/platform/endpoints/", "title": "Mistral AI docs — Benchmark results (archived, Feb 2024)", "quote": "Mistral Small 72.2% 86.9% 84.7% 86.9% 79.5% 51.7% Mistral Medium 75.3% 88.0% 88% 89.9% 81.1% 47% Mistral Large 81.2% 89.2% 86.7% 94.0% 82.7% 50.6%", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Same row/quote already used for this release's MMLU score. Table columns: MMLU, HellaSwag (10-shot), WinoGrande (5-shot), ARC-Challenge (25-shot), TriviaQA (5-shot), TruthfulQA." } }, { "benchmark": "arc-challenge", "value": 94, "config": "25-shot", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20240229181230/https://docs.mistral.ai/platform/endpoints/", "title": "Mistral AI docs — Benchmark results (archived, Feb 2024)", "quote": "Mistral Small 72.2% 86.9% 84.7% 86.9% 79.5% 51.7% Mistral Medium 75.3% 88.0% 88% 89.9% 81.1% 47% Mistral Large 81.2% 89.2% 86.7% 94.0% 82.7% 50.6%", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z", "note": "Same row/quote already used for this release's MMLU score. Table columns: MMLU, HellaSwag (10-shot), WinoGrande (5-shot), ARC-Challenge (25-shot), TriviaQA (5-shot), TruthfulQA." } } ] }, { "id": "anthropic-claude-3-opus", "lab": "anthropic", "name": "Claude 3 Opus", "family": "Claude Opus", "date": "2024-03-04", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-3-family", "title": "Introducing the next generation of Claude", "quote": "Opus and Sonnet are available to use today in our API, which is now generally available, enabling developers to sign up and start using these models immediately.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "retrieved_at": "2026-09-07T05:00:00Z" } ], "scores": [ { "benchmark": "mmlu", "value": 86.8, "config": "5-shot", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "MMLU 5-shot 86.8% 79.0% 75.2%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 1, columns Opus/Sonnet/Haiku; 86.8% is Opus's value." } }, { "benchmark": "humaneval", "value": 84.9, "config": "0-shot", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "0-shot 84.9% 73.0% 75.9%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 1 HumanEval row, columns Opus/Sonnet/Haiku; 84.9% is Opus's value." } }, { "benchmark": "math", "value": 60.1, "config": "0-shot", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "0-shot 60.1% 43.1% 38.9%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 1 MATH row (0-shot variant), columns Opus/Sonnet/Haiku; 60.1% is Opus's value. A 4-shot CoT variant is also reported at 61%." } }, { "benchmark": "gpqa-diamond", "value": 50.4, "config": "0-shot CoT", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "0-shotCoT 50.4% 40.4% 33.3%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Section 5.1 / Table 1 GPQA Diamond row, columns Opus/Sonnet/Haiku; 50.4% is Opus's value. With Maj@32 5-shot CoT, Opus scores 59.5%." } }, { "benchmark": "gsm8k", "value": 95, "config": "0-shot chain-of-thought", "note": "PDF source; the model card PDF fails to render through the r.jina.ai reader used for automated quote verification (repeated attempts returned an empty title/body), so this text was extracted locally with pdftotext instead and cannot yet be auto-verified. Table columns Opus/Sonnet/Haiku/GPT-4/GPT-3.5/Gemini 1.0 Ultra/1.5 Pro/1.0 Pro; 95.0% is Opus's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "GSM8K 95.0% 92.3% 88.9% 92.0% 57.1% 94.4% 91.7% 86.5%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "arc-challenge", "value": 96.4, "config": "25-shot", "note": "PDF source; see the note on this release's GSM8K score above regarding r.jina.ai fetch failures for this PDF. 96.4% is Opus's value in the common-sense-reasoning table.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "ARC-Challenge 25-shot 96.4%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "hellaswag", "value": 95.4, "config": "10-shot", "note": "PDF source; see the note on this release's GSM8K score above regarding r.jina.ai fetch failures for this PDF. 95.4% is Opus's value.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "HellaSwag 10-shot 95.4%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "winogrande", "value": 88.5, "config": "5-shot", "note": "PDF source; see the note on this release's GSM8K score above regarding r.jina.ai fetch failures for this PDF. Opus is the only model in this table's comparison for which WinoGrande is reported.", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "quote": "WinoGrande 5-shot 88.5%", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "lmarena-text", "value": 1322, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "242 230 256 claude-3-opus-20240229 Anthropic · Proprietary 1322 ±3 194,909", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Model card PDF is the primary source for scores; the marketing announcement page renders its benchmark comparison as an image chart with no extractable text. GSM8K/ARC-Challenge/HellaSwag/WinoGrande legacy scores added from the same model card's common-sense-reasoning and math tables (PDF source, not yet auto-verifiable — see per-score notes)." }, { "id": "xai-grok-1.5", "lab": "xai", "name": "Grok-1.5", "family": "Grok", "date": "2024-03-28", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok-1.5", "title": "Announcing Grok-1.5", "quote": "Introducing Grok-1.5, our latest model capable of long context understanding and advanced reasoning. Grok-1.5 will be available to our early testers and existing Grok users on the 𝕏 platform in the coming days.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" }, "scores": [ { "benchmark": "mmlu", "value": 81.3, "config": "5-shot", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-1.5", "title": "Announcing Grok-1.5", "quote": "| MMLU | 73% 5-shot | 81.3% 5-shot | 81.2% 5-shot | 75% 5-shot | 79% 5-shot | 83.7% 5-shot | 86.4% 5-shot | 86.8 5-shot |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "humaneval", "value": 74.1, "config": "0-shot", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-1.5", "title": "Announcing Grok-1.5", "quote": "it scored 74.1% on the HumanEval benchmark, which evaluates code generation and problem-solving abilities.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "math", "value": 50.6, "config": "4-shot", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-1.5", "title": "Announcing Grok-1.5", "quote": "Grok-1.5 achieved a 50.6% score on the MATH benchmark and a 90% score on the GSM8K benchmark", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "gsm8k", "value": 90, "reported_by": "official", "source": { "url": "https://x.ai/news/grok-1.5", "title": "Announcing Grok-1.5", "quote": "Grok-1.5 achieved a 50.6% score on the MATH benchmark and a 90% score on the GSM8K benchmark", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z", "note": "Same sentence already used for this release's MATH score." } } ], "notes": "Rolled out gradually to early testers and existing 𝕏 users starting Mar 28, 2024 (text-only; the multimodal 'Grok-1.5 Vision' preview followed Apr 12, 2024 as a separate, narrower preview). GPQA and MMMU not reported." }, { "id": "google-gemini-1.5-pro", "lab": "google", "name": "Gemini 1.5 Pro", "family": "Gemini Pro", "date": "2024-04-09", "date_precision": "day", "status": "released", "announcement": { "url": "https://developers.googleblog.com/2024/04/gemini-15-pro-in-public-preview-with-new-features.html", "title": "Gemini 1.5 Pro Now Available in 180+ Countries; with Native Audio Understanding, System Instructions, JSON Mode and more", "quote": "Today, we’re making Gemini 1.5 Pro available in 180+ countries via the Gemini API in public preview", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "sources": [ { "url": "https://blog.google/technology/ai/google-gemini-next-generation-model-february-2024/", "title": "Our next-generation model: Gemini 1.5", "quote": "we’re offering a limited preview of 1.5 Pro to developers and enterprise customers via AI Studio and Vertex AI", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z", "note": "Feb 15, 2024 announcement of 1.5 Pro; initially a private, waitlisted preview only." }, { "url": "https://blog.google/products-and-platforms/products/gemini/google-gemini-update-may-2024/", "title": "Get more done with Gemini: Try 1.5 Pro and more intelligent features", "quote": "Today we’re bringing Gemini 1.5 Pro, our cutting-edge model, to Gemini Advanced.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z", "note": "May 14, 2024: 1.5 Pro added to the consumer Gemini Advanced product." } ], "scores": [ { "benchmark": "mmlu", "value": 85.9, "config": "5-shot", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context", "quote": "67.7 (+9.2) 46.2 (+4.7) 89.2 (+5.2) 85.9 (+4.0) 84.1 (+12.2)", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "humaneval", "value": 84.1, "config": "0-shot", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context", "quote": "67.7 (+9.2) 46.2 (+4.7) 89.2 (+5.2) 85.9 (+4.0) 84.1 (+12.2)", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "math", "value": 67.7, "config": "4-shot", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context", "quote": "1.5 Pro’s performance on MATH (Hendrycks et al., 2021b) has improved from 58.5% to 67.7% while on GPQA (Rein et al., 2023) 1.5 Pro now scores 46.2% compared to 41.5% before.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "gpqa-diamond", "value": 46.2, "config": "4-shot", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context", "quote": "1.5 Pro’s performance on MATH (Hendrycks et al., 2021b) has improved from 58.5% to 67.7% while on GPQA (Rein et al., 2023) 1.5 Pro now scores 46.2% compared to 41.5% before.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "mmmu", "value": 62.2, "config": "0-shot", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context", "quote": "Gemini 1.5 Pro scores 62.2% on MMMU, improving over Gemini 1.0 Ultra.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "gsm8k", "value": 90.8, "config": "11-shot", "note": "Table 11, columns Gemini 1.0 Pro / 1.0 Ultra / 1.5 Flash / 1.5 Pro; 90.8% (11-shot) is 1.5 Pro's value, the fourth number in the quoted sequence.", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context", "quote": "GSM8K: Grade-school math problems. (Cobbe et al., 2021) 77.9% 11-shot 88.9% 11-shot 86.2% 11-shot 90.8% 11-shot", "retrieved_at": "2026-09-07T09:30:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } } ], "notes": "1.5 Pro shipped Feb 15, 2024 as a waitlisted private preview, then became broadly public via the Gemini API (180+ countries, no waitlist) on Apr 9, 2024 — recorded here as the release date — and was added to the consumer Gemini Advanced product May 14, 2024. Scores above are the improved 'Gemini 1.5 Pro (May 2024)' revision figures from the technical report, served under the same model name/endpoint as the April API rollout; the initial Feb 2024 revision scored lower on every one of these benchmarks (e.g. MATH 58.5%, GPQA 41.5%, MMLU ~78.9%). GSM8K legacy score added from the same report's Table 11 (11-shot)." }, { "id": "meta-llama-3-70b", "lab": "meta", "name": "Llama 3 70B", "family": "Llama", "date": "2024-04-18", "date_precision": "day", "status": "released", "announcement": { "url": "https://ai.meta.com/blog/meta-llama-3/", "title": "Introducing Meta Llama 3: The most capable openly available LLM to date", "quote": "Today, we're excited to share the first two models of the next generation of Llama, Meta Llama 3, available for broad use. This release features pretrained and instruction-fine-tuned language models with 8B and 70B parameters.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "mmlu", "value": 82, "config": "5-shot, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3/MODEL_CARD.md", "title": "Llama 3 Model Card (Meta)", "quote": "MMLU (5-shot) | 68.4 | 34.1 | 47.8 | 82.0 | 52.9", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "gpqa-diamond", "value": 39.5, "config": "0-shot, Instruct model (variant not specified as Diamond by Meta; recorded as-is)", "note": "Meta's card lists this simply as \"GPQA\" (0-shot), not explicitly \"GPQA Diamond\".", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3/MODEL_CARD.md", "title": "Llama 3 Model Card (Meta)", "quote": "GPQA (0-shot) | 34.2 | 21.7 | 22.3 | 39.5 | 21.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "humaneval", "value": 81.7, "config": "0-shot, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3/MODEL_CARD.md", "title": "Llama 3 Model Card (Meta)", "quote": "HumanEval (0-shot) | 62.2 | 7.9 | 14.0 | 81.7 | 25.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "math", "value": 50.4, "config": "4-shot, CoT, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3/MODEL_CARD.md", "title": "Llama 3 Model Card (Meta)", "quote": "MATH (4-shot, CoT) | 30.0 | 3.8 | 6.7 | 50.4 | 11.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "gsm8k", "value": 93, "config": "8-shot, CoT, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3/MODEL_CARD.md", "title": "Llama 3 Model Card (Meta)", "quote": "GSM-8K (8-shot, CoT)79.6 25.7 41.2 93.0 57.5", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Instruct-model benchmark table row. Columns: Llama 3 8B, Llama 2 7B, Llama 2 13B, Llama 3 70B (this value), Llama 2 70B." } }, { "benchmark": "lmarena-text", "value": 1276, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "289 281 295 llama-3-70b-instruct Meta · Llama 3 Community 1276 ±4 156,876", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "alibaba-qwen1.5-110b", "lab": "alibaba", "name": "Qwen1.5-110B", "family": "Qwen", "date": "2024-04-25", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen1.5-110b/", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "quote": "Today, we release the first 100B+ model of the Qwen1.5 series, Qwen1.5-110B, which achieves comparable performance with Meta-Llama3-70B in the base model evaluation, and outstanding performance in the chat evaluation.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:40Z" }, "scores": [ { "benchmark": "mmlu", "value": 80.4, "config": "base model", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen1.5-110b/", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "quote": "| MMLU | 80.4 | 77.5 | 79.5 | 77.8 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Qwen1.5-110B, Qwen1.5-72B, Llama-3-70B, Mixtral-8x22B." } }, { "benchmark": "hellaswag", "value": 87.5, "config": "base model", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen1.5-110b/", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "quote": "| Hellaswag | 87.5 | 86.0 | 88.0 | 88.7 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Qwen1.5-110B, Qwen1.5-72B, Llama-3-70B, Mixtral-8x22B." } }, { "benchmark": "arc-challenge", "value": 69.6, "config": "base model", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen1.5-110b/", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "quote": "| ARC-C | 69.6 | 65.9 | 68.8 | 70.7 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Qwen1.5-110B, Qwen1.5-72B, Llama-3-70B, Mixtral-8x22B." } }, { "benchmark": "gsm8k", "value": 85.4, "config": "base model", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen1.5-110b/", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "quote": "| GSM8K | 85.4 | 79.5 | 79.2 | 78.6 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Qwen1.5-110B, Qwen1.5-72B, Llama-3-70B, Mixtral-8x22B." } }, { "benchmark": "math", "value": 49.6, "config": "base model", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen1.5-110b/", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "quote": "| MATH | 49.6 | 34.1 | 41.0 | 41.7 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "humaneval", "value": 52.4, "config": "base model", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen1.5-110b/", "title": "Qwen1.5-110B: The First 100B+ Model of the Qwen1.5 Series", "quote": "| HumanEval | 52.4 | 41.5 | 45.7 | 45.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "lmarena-text", "value": 1234, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "312 307 322 qwen1.5-110b-chat Alibaba · Qianwen LICENSE 1234 ±6 26,195", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "deepseek-v2", "lab": "deepseek", "name": "DeepSeek-V2", "family": "DeepSeek-V", "date": "2024-05-06", "date_precision": "day", "status": "released", "announcement": { "url": "https://github.com/deepseek-ai/DeepSeek-V2", "title": "DeepSeek-V2", "quote": "2024.05.06: We released the DeepSeek-V2.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "scores": [ { "benchmark": "mmlu", "value": 77.8, "config": "Chat (RL)", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2/raw/main/README.md", "title": "DeepSeek-V2 model card", "quote": "| **MMLU** | English | 76.2 | 77.8 | 80.3 | 71.1 | 78.4 | 77.8 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "humaneval", "value": 81.1, "config": "Chat (RL)", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2/raw/main/README.md", "title": "DeepSeek-V2 model card", "quote": "| **HumanEval** | Code | 68.9 | 75.0 | 76.2 | 73.8 | 76.8 | 81.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "math", "value": 53.9, "config": "Chat (RL)", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2/raw/main/README.md", "title": "DeepSeek-V2 model card", "quote": "| **Math** | Math | 40.6 | 49.8 | 48.5 | 32.6 | 52.7 | 53.9 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hellaswag", "value": 87.1, "config": "10-shot, Acc., Base model", "note": "DeepSeek-V2's own README/paper do not report this benchmark; value taken from the base-model baseline comparison table in the later DeepSeek-V3 README (first data column, labeled 'DeepSeek-V2').", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card (base-model comparison table)", "quote": "| | HellaSwag (Acc.) | 10-shot | 87.1 | 84.8 | **89.2** | 88.9 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "arc-challenge", "value": 92.2, "config": "25-shot, Acc., Base model", "note": "DeepSeek-V2's own README/paper do not report this benchmark; value taken from the base-model baseline comparison table in the later DeepSeek-V3 README (first data column, labeled 'DeepSeek-V2').", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card (base-model comparison table)", "quote": "| | ARC-Challenge (Acc.) | 25-shot | 92.2 | 94.5 | **95.3** | **95.3** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "winogrande", "value": 86.3, "config": "5-shot, Acc., Base model", "note": "DeepSeek-V2's own README/paper do not report this benchmark; value taken from the base-model baseline comparison table in the later DeepSeek-V3 README (first data column, labeled 'DeepSeek-V2').", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card (base-model comparison table)", "quote": "| | WinoGrande (Acc.) | 5-shot | **86.3** | 82.3 | 85.2 | 84.9 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gsm8k", "value": 81.6, "config": "8-shot, EM, Base model", "note": "DeepSeek-V2's own README/paper do not report this benchmark; value taken from the base-model baseline comparison table in the later DeepSeek-V3 README (first data column, labeled 'DeepSeek-V2').", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card (base-model comparison table)", "quote": "| Math | GSM8K (EM) | 8-shot | 81.6 | 88.3 | 83.5 | **89.3** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } } ] }, { "id": "openai-gpt-4o", "lab": "openai", "name": "GPT-4o", "family": "GPT", "date": "2024-05-13", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/hello-gpt-4o/", "title": "Hello GPT-4o", "quote": "GPT‑4o's text and image capabilities are starting to roll out today in ChatGPT.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "mmlu", "value": 88.7, "config": "0-shot, chart from launch post", "note": "Value read from the SVG accessibility label (aria-label) of the MMLU bar in the launch post's benchmark chart (class \"bar-mmlu\").", "reported_by": "official", "source": { "url": "https://openai.com/index/hello-gpt-4o/", "quote": "GPT-4o, Accuracy (%): 88.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "humaneval", "value": 90.2, "config": "chart from launch post", "note": "Value read from the SVG accessibility label of the HumanEval bar (class \"bar-humaneval\").", "reported_by": "official", "source": { "url": "https://openai.com/index/hello-gpt-4o/", "quote": "GPT-4o, Accuracy (%): 90.2", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "math", "value": 76.6, "config": "chart from launch post", "note": "Value read from the SVG accessibility label of the MATH bar (class \"bar-math\").", "reported_by": "official", "source": { "url": "https://openai.com/index/hello-gpt-4o/", "quote": "GPT-4o, Accuracy (%): 76.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gpqa-diamond", "value": 53.6, "config": "reported simply as \"GPQA\"", "note": "OpenAI's chart labels this bar just \"GPQA\"; recorded here as the Diamond subset per standard convention at the time. Value read from the SVG accessibility label of the GPQA bar (class \"bar-gpqa\").", "reported_by": "official", "source": { "url": "https://openai.com/index/hello-gpt-4o/", "quote": "GPT-4o, Accuracy (%): 53.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1346, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "211 196 221 gpt-4o-2024-05-13 OpenAI · Proprietary 1346 ±3 112,881", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Launch date (May 13, 2024, OpenAI Spring Update) is well documented in press coverage but not stated in the post's own body text. Benchmark values are not present as page text; they were recovered from aria-label accessibility attributes on the SVG bar chart in the page's raw HTML, which are part of the page as served." }, { "id": "alibaba-qwen2-72b", "lab": "alibaba", "name": "Qwen2-72B", "family": "Qwen", "date": "2024-06-07", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "After months of efforts, we are pleased to announce the evolution from Qwen1.5 to Qwen2.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" }, "scores": [ { "benchmark": "mmlu", "value": 82.3, "config": "Instruct", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "English MMLU 82.0 75.6 82.3", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Llama-3-70B-Instruct, Qwen1.5-72B-Chat, Qwen2-72B-Instruct." } }, { "benchmark": "mmlu-pro", "value": 64.4, "config": "Instruct", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "MMLU-Pro 56.2 51.7 64.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "humaneval", "value": 86, "config": "Instruct", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "Coding HumanEval 81.7 71.3 86.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "math", "value": 59.7, "config": "Instruct", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "MATH 50.4 42.5 59.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "gpqa-diamond", "value": 42.4, "config": "Instruct", "note": "Lab reports this simply as 'GPQA'; recorded as GPQA Diamond per standard practice, though the source page does not use the word 'Diamond'.", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "GPQA 41.9 39.4 42.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "gsm8k", "value": 91.1, "config": "Instruct", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "| GSM8K | **93.0** | 82.7 | 91.1 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table columns: Llama-3-70B-Instruct, Qwen1.5-72B-Chat, Qwen2-72B-Instruct." } }, { "benchmark": "hellaswag", "value": 87.6, "config": "Base model", "note": "HellaSwag is only reported for the Base model in this post's appendix comparison table, not for the Instruct model used for the other legacy/index scores above.", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "| HellaSwag | 87.8 | **88.7** | 88.0 | 86.0 | 87.5 | 87.6 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Appendix 'Base Language Model Evaluation' table; columns: DeepSeek-V2, Mixtral-8x22B, Llama-3-70B, Qwen1.5-72B, Qwen1.5-110B, Qwen2-72B." } }, { "benchmark": "winogrande", "value": 85.1, "config": "Base model", "note": "WinoGrande is only reported for the Base model in this post's appendix comparison table, not for the Instruct model used for the other legacy/index scores above. Source spells it 'WindoGrande' (typo in the original).", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "| WindoGrande | 84.8 | 85.0 | **85.3** | 83.0 | 83.5 | 85.1 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Appendix 'Base Language Model Evaluation' table; columns: DeepSeek-V2, Mixtral-8x22B, Llama-3-70B, Qwen1.5-72B, Qwen1.5-110B, Qwen2-72B." } }, { "benchmark": "arc-challenge", "value": 68.9, "config": "Base model", "note": "ARC-Challenge is only reported for the Base model in this post's appendix comparison table, not for the Instruct model used for the other legacy/index scores above.", "reported_by": "official", "source": { "url": "https://qwenlm.github.io/blog/qwen2/", "title": "Hello Qwen2", "quote": "| ARC-C | 70.0 | **70.7** | 68.8 | 65.9 | 69.6 | 68.9 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Appendix 'Base Language Model Evaluation' table; columns: DeepSeek-V2, Mixtral-8x22B, Llama-3-70B, Qwen1.5-72B, Qwen1.5-110B, Qwen2-72B." } }, { "benchmark": "lmarena-text", "value": 1261, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "299 293 306 qwen2-72b-instruct Alibaba · Qianwen LICENSE 1261 ±5 37,325", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "anthropic-claude-3.5-sonnet", "lab": "anthropic", "name": "Claude 3.5 Sonnet", "family": "Claude Sonnet", "date": "2024-06-20", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-3-5-sonnet", "title": "Introducing Claude 3.5 Sonnet", "quote": "Today, we're launching Claude 3.5 Sonnet—our first release in the forthcoming Claude 3.5 model family.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf", "title": "Claude 3.5 Sonnet Model Card Addendum", "retrieved_at": "2026-09-07T05:00:00Z" } ], "scores": [ { "benchmark": "gpqa-diamond", "value": 59.4, "config": "0-shot CoT", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf", "title": "Claude 3.5 Sonnet Model Card Addendum", "quote": "GPQA(Diamond) 0-shotCoT 59.4% 50.4% 40.4% 53.6% 48.0%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 1, columns Claude 3.5 Sonnet / Claude 3 Opus / Claude 3 Sonnet / GPT-4o / GPT-4T; 59.4% is 3.5 Sonnet's value." } }, { "benchmark": "mmlu", "value": 88.7, "config": "5-shot", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf", "title": "Claude 3.5 Sonnet Model Card Addendum", "quote": "5-shot 88.7% 86.8% 78.3%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 1 MMLU 5-shot row; 88.7% is 3.5 Sonnet's value." } }, { "benchmark": "math", "value": 71.1, "config": "0-shot CoT", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf", "title": "Claude 3.5 Sonnet Model Card Addendum", "quote": "MATH[10] 71.1% 60.1% 43.1% 76.6% 72.6%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 1 MATH row; 71.1% is 3.5 Sonnet's value." } }, { "benchmark": "humaneval", "value": 92, "config": "0-shot", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf", "title": "Claude 3.5 Sonnet Model Card Addendum", "quote": "0-shot 92.0% 84.9% 73.0% 90.2% 87.1%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 1 HumanEval row; 92.0% is 3.5 Sonnet's value." } }, { "benchmark": "mmmu", "value": 68.3, "config": "validation split, 0-shot CoT", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf", "title": "Claude 3.5 Sonnet Model Card Addendum", "quote": "MMMU[18](validation) 68.3% 59.4% 53.1% 69.1% 63.1%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 2, columns 3.5 Sonnet / Claude 3 Opus / Claude 3 Sonnet / GPT-4o / GPT-4T." } }, { "benchmark": "lmarena-text", "value": 1374, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "179 171 184 claude-3-5-sonnet-20241022 Anthropic · Proprietary 1374 ±3 87,696", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "This is the original June 2024 Claude 3.5 Sonnet (model string claude-3-5-sonnet-20240620), the top tier at the time (Claude 3 Opus successor was not yet released). A later 'upgraded' Claude 3.5 Sonnet (October 2024, claude-3-5-sonnet-20241022) is treated as an incremental update, not a separate flagship entry." }, { "id": "meta-llama-3.1-405b", "lab": "meta", "name": "Llama 3.1 405B", "family": "Llama", "date": "2024-07-23", "date_precision": "day", "status": "released", "announcement": { "url": "https://ai.meta.com/blog/meta-llama-3-1/", "title": "Introducing Llama 3.1: Our most capable models to date", "quote": "Llama 3.1 405B is the first openly available model that rivals the top AI models when it comes to state-of-the-art capabilities in general knowledge, steerability, math, tool use, and multilingual translation.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "mmlu", "value": 87.3, "config": "5-shot, macro_avg/acc, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md", "title": "Llama 3.1 Model Card (Meta)", "quote": "General | MMLU | 5 | macro_avg/acc | 68.5 | 69.4 | 82.0 | 83.6 | 87.3", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "gpqa-diamond", "value": 50.7, "config": "0-shot, em, Instruct model (variant not specified as Diamond by Meta; recorded as-is)", "note": "Meta's card lists this simply as \"GPQA\" (0-shot), not explicitly \"GPQA Diamond\".", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md", "title": "Llama 3.1 Model Card (Meta)", "quote": "GPQA | 0 | em | 34.6 | 30.4 | 39.5 | 46.7 | 50.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "humaneval", "value": 89, "config": "0-shot, pass@1, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md", "title": "Llama 3.1 Model Card (Meta)", "quote": "Code | HumanEval | 0 | pass@1 | 60.4 | 72.6 | 81.7 | 80.5 | 89.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "math", "value": 73.8, "config": "0-shot, CoT, final_em, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md", "title": "Llama 3.1 Model Card (Meta)", "quote": "MATH (CoT) | 0 | final_em | 29.1 | 51.9 | 51.0 | 68.0 | 73.8", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "gsm8k", "value": 96.8, "config": "8-shot, em_maj1@1, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md", "title": "Llama 3.1 Model Card (Meta)", "quote": "Math GSM-8K (CoT)8 em_maj1@1 80.6 84.5 93.0 95.1 96.8", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Instruct-model benchmark table row. Columns: Llama 3 8B, Llama 3.1 8B, Llama 3 70B, Llama 3.1 70B, Llama 3.1 405B (this value)." } }, { "benchmark": "arc-challenge", "value": 96.9, "config": "0-shot, acc, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md", "title": "Llama 3.1 Model Card (Meta)", "quote": "Reasoning ARC-C 0 acc 82.4 83.4 94.4 94.8 96.9", "retrieved_at": "2026-09-07T10:00:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "note": "Instruct-model benchmark table row ('ARC-C'). Columns: Llama 3 8B, Llama 3.1 8B, Llama 3 70B, Llama 3.1 70B, Llama 3.1 405B (this value)." } } ] }, { "id": "mistral-mistral-large-2", "lab": "mistral", "name": "Mistral Large 2", "family": "Mistral Large", "date": "2024-07-24", "date_precision": "day", "status": "released", "announcement": { "url": "https://mistral.ai/news/mistral-large-2407/", "title": "Large Enough", "quote": "Mistral Large 2 is exposed on la Plateforme and enriched with new features to facilitate building innovative AI applications.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" }, "scores": [ { "benchmark": "mmlu", "value": 84, "config": "pretrained model", "reported_by": "official", "source": { "url": "https://mistral.ai/news/mistral-large-2407/", "title": "Large Enough", "quote": "on MMLU, the pretrained version achieves an accuracy of 84.0%", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" } } ], "notes": "Mistral Large 2411 (the Nov 2024 24.11 point release, 'Mistral Large 2.1') is not recorded as a separate entry: it shipped alongside the Pixtral Large announcement without its own dedicated launch post, only a model-card page. GSM8K, MATH and HumanEval scores could not be added: the launch post ('Large Enough') renders its code, reasoning and math benchmark results only as chart images with no extractable numeric text (re-checked 2026-09-07), and the gated Hugging Face model card (mistralai/Mistral-Large-Instruct-2407) requires authentication so its README could not be fetched anonymously." }, { "id": "xai-grok-2", "lab": "xai", "name": "Grok-2", "family": "Grok", "date": "2024-08-13", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok-2", "title": "Grok-2 Beta Release", "quote": "We are excited to release an early preview of Grok-2, a significant step forward from our previous model Grok-1.5, featuring frontier capabilities in chat, coding, and reasoning.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" }, "scores": [ { "benchmark": "gpqa-diamond", "value": 56, "config": "0-shot CoT", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-2", "title": "Grok-2 Beta Release", "quote": "| GPQA | | 35.9% | 51.0% | 56.0% | 48.0% | 50.4% | 46.2% | 51.1% | 53.6% | 59.6% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "mmlu", "value": 87.5, "reported_by": "official", "source": { "url": "https://x.ai/news/grok-2", "title": "Grok-2 Beta Release", "quote": "| MMLU | | 81.3% | 86.2% | 87.5% | 86.5% | 85.7% | 85.9% | 88.6% | 88.7% | 88.3% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "mmlu-pro", "value": 75.5, "config": "0-shot CoT", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-2", "title": "Grok-2 Beta Release", "quote": "| MMLU-Pro | | 51.0% | 72.0% | 75.5% | 63.7% | 68.5% | 69.0% | 73.3% | 72.6% | 76.1% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "math", "value": 76.1, "config": "maj@1", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-2", "title": "Grok-2 Beta Release", "quote": "| MATH§ | | 50.6% | 73.0% | 76.1% | 72.6% | 60.1% | 67.7% | 73.8% | 76.6% | 71.1% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "humaneval", "value": 88.4, "config": "pass@1", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-2", "title": "Grok-2 Beta Release", "quote": "| HumanEval¶ | | 74.1% | 85.7% | 88.4% | 87.1% | 84.9% | 71.9% | 89.0% | 90.2% | 92.0% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "mmmu", "value": 66.1, "config": "0-shot CoT", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-2", "title": "Grok-2 Beta Release", "quote": "| MMMU | | 53.6% | 63.2% | 66.1% | 63.1% | 59.4% | 62.2% | 64.5% | 69.1% | 68.3% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "lmarena-text", "value": 1335, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "225 207 234 grok-2-2024-08-13 SpaceXAI · Proprietary 1335 ±4 63,498", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Launched Aug 13, 2024 as a beta on 𝕏 (as 'sus-column-r' on the LMArena leaderboard before the name reveal); the enterprise API followed later that month. Benchmark table columns confirmed by position: Grok-1.5, Grok-2 mini, Grok-2 (used here), GPT-4 Turbo, Claude 3 Opus, Gemini Pro 1.5, Llama 3 405B, GPT-4o, Claude 3.5 Sonnet." }, { "id": "zhipu-glm-4-plus", "lab": "zhipu", "name": "GLM-4-Plus", "family": "GLM", "date": "2024-08-29", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.zhipuai.cn/zh/news/69", "title": "智谱携基座大模型GLM-4-Plus 亮相KDD", "quote": "在 KDD 国际数据挖掘与知识发现大会上，智谱 GLM 团队介绍了新一代基座大模型GLM-4-Plus。", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z" }, "scores": [ { "benchmark": "mmlu", "value": 86.8, "reported_by": "official", "source": { "url": "https://docs.bigmodel.cn/cn/guide/models/text/glm-4", "title": "GLM-4 系列模型 — 智谱AI开放平台文档", "quote": "| GLM-4-Plus | 83.2 | 86.8 | 74.2 | 50.7 | *45.8 | 50.4 | 79.5 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Row columns: AlignBench, MMLU, MATH, GPQA, LiveCodeBench(*reproduced by Zhipu), NaturalCodeBench, IFEval." } }, { "benchmark": "math", "value": 74.2, "reported_by": "official", "source": { "url": "https://docs.bigmodel.cn/cn/guide/models/text/glm-4", "title": "GLM-4 系列模型 — 智谱AI开放平台文档", "quote": "| GLM-4-Plus | 83.2 | 86.8 | 74.2 | 50.7 | *45.8 | 50.4 | 79.5 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Row columns: AlignBench, MMLU, MATH, GPQA, LiveCodeBench(*reproduced by Zhipu), NaturalCodeBench, IFEval." } }, { "benchmark": "gpqa-diamond", "value": 50.7, "note": "Column labeled 'GPQA' in source; Diamond subset not explicitly specified.", "reported_by": "official", "source": { "url": "https://docs.bigmodel.cn/cn/guide/models/text/glm-4", "title": "GLM-4 系列模型 — 智谱AI开放平台文档", "quote": "| GLM-4-Plus | 83.2 | 86.8 | 74.2 | 50.7 | *45.8 | 50.4 | 79.5 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Row columns: AlignBench, MMLU, MATH, GPQA, LiveCodeBench(*reproduced by Zhipu), NaturalCodeBench, IFEval." } }, { "benchmark": "livecodebench", "value": 45.8, "config": "reproduced by Zhipu (marked * in source table)", "reported_by": "official", "source": { "url": "https://docs.bigmodel.cn/cn/guide/models/text/glm-4", "title": "GLM-4 系列模型 — 智谱AI开放平台文档", "quote": "| GLM-4-Plus | 83.2 | 86.8 | 74.2 | 50.7 | *45.8 | 50.4 | 79.5 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Row columns: AlignBench, MMLU, MATH, GPQA, LiveCodeBench(*reproduced by Zhipu), NaturalCodeBench, IFEval." } }, { "benchmark": "lmarena-text", "value": 1319, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "246 230 261 glm-4-plus Z.ai · Proprietary 1319 ±5 26,126", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "deepseek-v2.5", "lab": "deepseek", "name": "DeepSeek-V2.5", "family": "DeepSeek-V", "date": "2024-09-05", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5/raw/main/README.md", "title": "DeepSeek-V2.5", "quote": "DeepSeek-V2.5 is an upgraded version that combines DeepSeek-V2-Chat and DeepSeek-Coder-V2-Instruct. The new model integrates the general and coding abilities of the two previous versions.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news250120", "title": "DeepSeek API news index (lists DeepSeek-V2.5 Release 2024/09/05)", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "humaneval", "value": 89, "config": "HumanEval python", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5/raw/main/README.md", "title": "DeepSeek-V2.5 model card", "quote": "| HumanEval python | 84.5 | 87.2 | 89 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "livecodebench", "value": 41.8, "config": "window 01-09", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5/raw/main/README.md", "title": "DeepSeek-V2.5 model card", "quote": "| LiveCodeBench(01-09) | 36.6 | 39.7 | 41.8 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1307, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "262 250 273 deepseek-v2.5 DeepSeek · DeepSeek 1307 ±5 24,572", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "No MMLU/GPQA/MATH figures were published in this model card (it focuses on deltas over DeepSeek-V2-Chat and DeepSeek-Coder-V2-Instruct)." }, { "id": "openai-o1-preview", "lab": "openai", "name": "OpenAI o1-preview", "family": "o-series", "date": "2024-09-12", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/introducing-openai-o1-preview/", "title": "Introducing OpenAI o1-preview", "quote": "Today, we are releasing the first of this series in ChatGPT and our API.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "swe-bench-verified", "value": 41.3, "config": "pre- and post-mitigation, o1-preview", "reported_by": "official", "source": { "url": "https://openai.com/index/openai-o1-system-card/", "title": "OpenAI o1 System Card", "quote": "o1‑preview (Pre-Mitigation and Post-Mitigation) performs the best on SWE-bench Verified at 41.3%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1388, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "162 149 173 o1-preview OpenAI · Proprietary 1388 ±5 31,122", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Public launch date (September 12, 2024) is stated on the announcement page itself. AIME/GPQA Diamond figures for o1-preview are shown only as image charts in both the launch post and the companion technical post (openai.com/index/learning-to-reason-with-llms/); no extractable text values were found for those within the research budget, so only the SWE-bench Verified figure (from the o1 system card's text) is recorded." }, { "id": "alibaba-qwen2.5-72b", "lab": "alibaba", "name": "Qwen2.5-72B", "family": "Qwen", "date": "2024-09-19", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen2.5/", "title": "Qwen2.5: A Party of Foundation Models!", "quote": "Today, we are excited to introduce the latest addition to the Qwen family: Qwen2.5.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" }, "scores": [ { "benchmark": "mmlu-pro", "value": 71.1, "config": "Instruct", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2412.15115v2", "title": "Qwen2.5 Technical Report, Table 6", "quote": "MMLU-Pro 66.4 73.3 64.4 71.1 72.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table 6 columns: Llama-3.1-70B, Llama-3.1-405B, Qwen2-72B, Qwen2.5-72B, Qwen2.5-Plus." } }, { "benchmark": "math", "value": 83.1, "config": "Instruct", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2412.15115v2", "title": "Qwen2.5 Technical Report, Table 6", "quote": "MATH 68.0 73.8 69.0 83.1 84.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "humaneval", "value": 86.6, "config": "Instruct", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2412.15115v2", "title": "Qwen2.5 Technical Report, Table 6", "quote": "HumanEval 80.5 89.0 86.0 86.6 87.8", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "livecodebench", "value": 55.5, "config": "LiveCodeBench 2305-2409, Instruct", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2412.15115v2", "title": "Qwen2.5 Technical Report, Table 6", "quote": "LiveCodeBench 32.1 41.6 32.2 55.5 51.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "gpqa-diamond", "value": 49, "config": "Instruct", "note": "Lab reports this simply as 'GPQA'; recorded as GPQA Diamond per standard practice, though the source does not use the word 'Diamond'.", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2412.15115v2", "title": "Qwen2.5 Technical Report, Table 6", "quote": "GPQA 46.7 51.1 42.4 49.0 49.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "gsm8k", "value": 95.8, "config": "Instruct", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2412.15115v2", "title": "Qwen2.5 Technical Report, Table 6", "quote": "GSM8K 95.1 96.8 93.2 95.8 96.0", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table 6 columns: Llama-3.1-70B, Llama-3.1-405B, Qwen2-72B, Qwen2.5-72B, Qwen2.5-Plus." } }, { "benchmark": "lmarena-text", "value": 1303, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "270 257 274 qwen2.5-72b-instruct Alibaba · Qwen 1303 ±4 39,406", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "deepseek-v3", "lab": "deepseek", "name": "DeepSeek-V3", "family": "DeepSeek-V", "date": "2024-12-26", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3", "quote": "We present DeepSeek-V3, a strong Mixture-of-Experts (MoE) language model with 671B total parameters with 37B activated for each token.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news250120", "title": "DeepSeek API news index (lists \"Introducing DeepSeek-V3\" 2024/12/26)", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu", "value": 88.5, "config": "Chat model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| English | MMLU (EM) | 78.2 | 80.6 | 85.3 | **88.6** | **88.3** | 87.2 | **88.5** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 59.1, "config": "pass@1, Chat model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| | GPQA-Diamond (Pass@1) | 35.3 | 41.3 | 49.0 | 51.1 | **65.0** | 49.9 | 59.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "math", "value": 90.2, "config": "MATH-500, EM, Chat model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| | MATH-500 (EM) | 56.3 | 74.7 | 80.0 | 73.8 | 78.3 | 74.6 | **90.2** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "aime", "value": 39.2, "config": "AIME 2024, pass@1, Chat model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| Math | AIME 2024 (Pass@1) | 4.6 | 16.7 | 23.3 | 23.3 | 16.0 | 9.3 | **39.2** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "humaneval", "value": 65.2, "config": "0-shot, pass@1, Base (pretrained) model", "note": "The Chat model card reports \"HumanEval-Mul\" (a multilingual variant) instead of plain HumanEval; this value is from the Base model table.", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| Code | HumanEval (Pass@1) | 0-shot | 43.3 | 53.0 | 54.9 | **65.2** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gsm8k", "value": 89.3, "config": "8-shot, EM, Base (pretrained) model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| Math | GSM8K (EM) | 8-shot | 81.6 | 88.3 | 83.5 | **89.3** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hellaswag", "value": 88.9, "config": "10-shot, Acc., Base (pretrained) model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| | HellaSwag (Acc.) | 10-shot | 87.1 | 84.8 | **89.2** | 88.9 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "arc-challenge", "value": 95.3, "config": "25-shot, Acc., Base (pretrained) model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| | ARC-Challenge (Acc.) | 25-shot | 92.2 | 94.5 | **95.3** | **95.3** |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "winogrande", "value": 84.9, "config": "5-shot, Acc., Base (pretrained) model", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3/raw/main/README.md", "title": "DeepSeek-V3 model card", "quote": "| | WinoGrande (Acc.) | 5-shot | **86.3** | 82.3 | 85.2 | 84.9 |", "retrieved_at": "2026-09-07T05:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1358, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "192 184 209 deepseek-v3 DeepSeek · DeepSeek 1358 ±5 21,770", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "deepseek-r1", "lab": "deepseek", "name": "DeepSeek-R1", "family": "DeepSeek-R", "date": "2025-01-20", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/raw/main/README.md", "title": "DeepSeek-R1", "quote": "We introduce our first-generation reasoning models, DeepSeek-R1-Zero and DeepSeek-R1.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://www.deepseek.com/en/news/deepseek-r1/", "title": "DeepSeek | DeepSeek-R1 Release", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu", "value": 90.8, "config": "pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/raw/main/README.md", "title": "DeepSeek-R1 model card", "quote": "| English | MMLU (Pass@1) | 88.3 | 87.2 | 88.5 | 85.2 | **91.8** | 90.8 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 71.5, "config": "pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/raw/main/README.md", "title": "DeepSeek-R1 model card", "quote": "| | GPQA-Diamond (Pass@1) | 65.0 | 49.9 | 59.1 | 60.0 | **75.7** | 71.5 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "math", "value": 97.3, "config": "MATH-500, pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/raw/main/README.md", "title": "DeepSeek-R1 model card", "quote": "| | MATH-500 (Pass@1) | 78.3 | 74.6 | 90.2 | 90.0 | 96.4 | **97.3** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "aime", "value": 79.8, "config": "AIME 2024, pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/raw/main/README.md", "title": "DeepSeek-R1 model card", "quote": "| Math | AIME 2024 (Pass@1) | 16.0 | 9.3 | 39.2 | 63.6 | 79.2 | **79.8** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "swe-bench-verified", "value": 49.2, "config": "Resolved", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1/raw/main/README.md", "title": "DeepSeek-R1 model card", "quote": "| | SWE Verified (Resolved) | **50.8** | 38.8 | 42.0 | 41.6 | 48.9 | 49.2 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1398, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "149 139 163 deepseek-r1 DeepSeek · MIT 1398 ±5 18,524", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "moonshot-kimi-k1.5", "lab": "moonshot", "name": "Kimi k1.5", "family": "Kimi K", "date": "2025-01-20", "date_precision": "day", "status": "released", "announcement": { "url": "https://github.com/MoonshotAI/Kimi-k1.5", "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs", "quote": "Introducing Kimi k1.5 --- an o1-level multi-modal model", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" }, "scores": [ { "benchmark": "math", "value": 96.2, "config": "MATH-500, long-CoT", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2501.12599v1", "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs", "quote": "MATH-500 (EM) 90.6 90.0 - 94.8 96.2 AIME 2024 (Pass@1) 50.0 63.6 - 74.4 77.5 Codeforces (Percentile) 62 88 - 94 94 LiveCodeBench (Pass@1) 40.6 53.1 - 67.2 62.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Table columns: QwQ-32B-Preview, OpenAI o1-mini, QVQ-72B-Preview, OpenAI o1, Kimi k1.5 (long-CoT)." } }, { "benchmark": "aime", "value": 77.5, "config": "AIME 2024, long-CoT, pass@1", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2501.12599v1", "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs", "quote": "MATH-500 (EM) 90.6 90.0 - 94.8 96.2 AIME 2024 (Pass@1) 50.0 63.6 - 74.4 77.5 Codeforces (Percentile) 62 88 - 94 94 LiveCodeBench (Pass@1) 40.6 53.1 - 67.2 62.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "livecodebench", "value": 62.5, "config": "long-CoT, pass@1", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2501.12599v1", "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs", "quote": "MATH-500 (EM) 90.6 90.0 - 94.8 96.2 AIME 2024 (Pass@1) 50.0 63.6 - 74.4 77.5 Codeforces (Percentile) 62 88 - 94 94 LiveCodeBench (Pass@1) 40.6 53.1 - 67.2 62.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } } ] }, { "id": "alibaba-qwen2.5-max", "lab": "alibaba", "name": "Qwen2.5-Max", "family": "Qwen", "date": "2025-01-28", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen2.5-max/", "title": "Qwen2.5-Max: Exploring the Intelligence of Large-scale MoE Model", "quote": "Today, we are excited to share the performance results of Qwen2.5-Max and announce the availability of its API through Alibaba Cloud.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 1374, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "178 170 185 qwen2.5-max Alibaba · Proprietary 1374 ±4 32,417", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "The launch post presents all benchmark comparisons (MMLU-Pro, LiveCodeBench, LiveBench, Arena-Hard, GPQA-Diamond) as chart images with no accompanying text table; no official numeric scores could be sourced within the research window. The lab's arXiv technical report (2412.15115) covers Qwen2.5 but predates and does not cover Qwen2.5-Max specifically." }, { "id": "google-gemini-2.0-pro-experimental", "lab": "google", "name": "Gemini 2.0 Pro Experimental", "family": "Gemini Pro", "date": "2025-02-05", "date_precision": "day", "status": "released", "announcement": { "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025/", "title": "Gemini 2.0 is now available to everyone", "quote": "We’re also releasing an experimental version of Gemini 2.0 Pro, our best model yet for coding performance and complex prompts.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "scores": [], "notes": "Released only as a free-standing 'Experimental' model (gemini-2.0-pro-exp-02-05) in Google AI Studio, Vertex AI and the Gemini app for Gemini Advanced users — never reached general availability and was superseded by Gemini 2.5 Pro about seven weeks later. Counted here as 'released' per project convention because it was publicly usable (no waitlist), not merely announced. Google did not publish a numeric academic-benchmark table alongside this launch (the post and companion Google for Developers post describe capability only qualitatively); no verifiable official score could be found for this specific checkpoint, so scores are left empty rather than guessed." }, { "id": "xai-grok-3", "lab": "xai", "name": "Grok 3", "family": "Grok", "date": "2025-02-19", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok-3", "title": "Grok 3 Beta — The Age of Reasoning Agents", "quote": "We are pleased to introduce Grok 3, our most advanced model yet: blending strong reasoning with extensive pretraining knowledge.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" }, "scores": [ { "benchmark": "gpqa-diamond", "value": 84.6, "config": "Grok 3 (Think) mode", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-3", "title": "Grok 3 Beta — The Age of Reasoning Agents", "quote": "Grok 3 (Think) also attained 84.6% on graduate-level expert reasoning (GPQA)", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "aime", "value": 93.3, "config": "AIME 2025, Grok 3 (Think) mode, cons@64", "note": "Highest test-time compute setting (64-sample majority vote / cons@64), not a plain pass@1 number.", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-3", "title": "Grok 3 Beta — The Age of Reasoning Agents", "quote": "With our highest level of test-time compute (cons@64), Grok 3 (Think) achieved 93.3% on this competition.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "livecodebench", "value": 79.4, "config": "Grok 3 (Think) mode, 10/1/2024–2/1/2025 window", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-3", "title": "Grok 3 Beta — The Age of Reasoning Agents", "quote": "and 79.4% on LiveCodeBench for code generation and problem-solving.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "mmlu-pro", "value": 79.9, "config": "Grok 3 Beta, non-reasoning", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-3", "title": "Grok 3 Beta — The Age of Reasoning Agents", "quote": "| MMLU-pro | 79.9% | 78.9% | 79.1% | 75.9% | 72.6% | 78.0% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "mmmu", "value": 73.2, "config": "Grok 3 Beta, non-reasoning", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-3", "title": "Grok 3 Beta — The Age of Reasoning Agents", "quote": "| MMMU | 73.2% | 69.4% | 72.7% | — | 69.1% | 70.4% |", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } } ], "notes": "Launched Feb 19, 2025 as an early preview/beta to 𝕏 Premium+ users and grok.com, with the API following weeks later; both models were explicitly 'still in training' at launch. GPQA/AIME/LiveCodeBench figures are for the Grok 3 (Think) reasoning mode included in this same release; MMLU-Pro/MMMU are from the separate non-reasoning 'Grok 3 Beta' pretraining table in the same post (no Think-mode figures given for those two)." }, { "id": "anthropic-claude-3.7-sonnet", "lab": "anthropic", "name": "Claude 3.7 Sonnet", "family": "Claude Sonnet", "date": "2025-02-24", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-3-7-sonnet", "title": "Claude 3.7 Sonnet and Claude Code", "quote": "Claude 3.7 Sonnet is now available on all Claude plans—including Free, Pro, Team, and Enterprise—as well as the Claude Developer Platform, Amazon Bedrock, and Google Cloud's Vertex AI.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 1388, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "163 151 173 claude-3-7-sonnet-20250219-thinking-32k Anthropic · Proprietary 1388 ±4 38,090", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "The launch post states 'Claude 3.7 Sonnet achieves state-of-the-art performance on SWE-bench Verified' and shows comparison charts for SWE-bench Verified, TAU-bench and a broader benchmark table, but all of these are rendered as images with no extractable text, and the companion system card (anthropic.com/claude-3-7-sonnet-system-card) covers only safety evaluations, not the standard capability benchmarks. No verified numeric score could be sourced within the research budget for this release; left empty rather than guessed. Widely-reported third-party figures (e.g. ~70.3% SWE-bench Verified, ~86.4% MMLU) exist but were not confirmed against Anthropic's own text and are therefore omitted." }, { "id": "openai-gpt-4.5", "lab": "openai", "name": "GPT-4.5", "family": "GPT", "date": "2025-02-27", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/introducing-gpt-4-5/", "title": "Introducing GPT-4.5", "quote": "We're releasing a research preview of GPT‑4.5—our largest and best model for chat yet.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "gpqa-diamond", "value": 71.4, "config": "reported as \"GPQA (science)\"", "note": "Table row 'GPQA (science)' with columns GPT-4.5 / GPT-4o / OpenAI o3-mini (high); 71.4% is GPT-4.5's value.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-4-5/", "quote": "GPQA (science)71.4%53.6%79.7%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "aime", "value": 36.7, "config": "AIME 2024", "note": "Table row 'AIME '24 (math)' with columns GPT-4.5 / GPT-4o / OpenAI o3-mini (high); 36.7% is GPT-4.5's value.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-4-5/", "quote": "AIME ‘24 (math)36.7%9.3%87.3%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 74.4, "config": "multimodal eval table", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-4-5/", "quote": "MMMU (multimodal)74.4%69.1%-", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "swe-bench-verified", "value": 38, "config": "best internal performance", "note": "Table footnote: 'Numbers shown represent best internal performance.'", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-4-5/", "quote": "SWE-Bench Verified (coding)*38.0%30.7%61.0%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1445, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "76 64 94 gpt-4.5-preview-2025-02-27 OpenAI · Proprietary 1445 ±6 14,547", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Available at launch to ChatGPT Pro subscribers and developers (research preview), which counts as public availability under this project's rules. Date corroborated by the OpenAI system card filename ('gpt-4-5-system-card-2272025.pdf') and CNBC's same-day report (https://www.cnbc.com/2025/02/27/openai-launching-gpt-4point5-general-purpose-large-language-model.html)." }, { "id": "google-gemini-2.5-pro", "lab": "google", "name": "Gemini 2.5 Pro", "family": "Gemini Pro", "date": "2025-03-25", "date_precision": "day", "status": "released", "announcement": { "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/", "title": "Gemini 2.5: Our most intelligent AI model", "quote": "Today we’re introducing Gemini 2.5, our most intelligent AI model. Our first 2.5 release is an experimental version of 2.5 Pro", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "sources": [ { "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Pro-Model-Card.pdf", "title": "Gemini 2.5 Pro Model Card", "retrieved_at": "2026-09-07T07:45:00Z", "note": "Contains a multi-version benchmark table (Experimental 03-25 / Preview 05-06 / GA); scores below use the 'Experimental 03-25' column matching this release's launch date." } ], "scores": [ { "benchmark": "hle", "value": 18.8, "config": "no tools", "reported_by": "official", "source": { "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/", "title": "Gemini 2.5: Our most intelligent AI model", "quote": "It also scores a state-of-the-art 18.8% across models without tool use on Humanity’s Last Exam", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "gpqa-diamond", "value": 84, "config": "single attempt (pass@1), Experimental 03-25", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Pro-Model-Card.pdf", "title": "Gemini 2.5 Pro Model Card", "quote": "GPQA diamond single attempt (pass@1) 86.4% 83.0% 84.0% 83.3% 81.4% 75.4% 79.6% 80.2% 81.0%", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "aime", "value": 86.7, "config": "AIME 2025, single attempt (pass@1), Experimental 03-25", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Pro-Model-Card.pdf", "title": "Gemini 2.5 Pro Model Card", "quote": "AIME 2025 single attempt (pass@1) 88.0% 83.0% 86.7% 88.9% 92.7% 70.5% 75.5% 77.3% 87.5%", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "swe-bench-verified", "value": 63.8, "config": "custom agent setup", "reported_by": "official", "source": { "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/", "title": "Gemini 2.5: Our most intelligent AI model", "quote": "On SWE-Bench Verified, the industry standard for agentic code evals, Gemini 2.5 Pro scores 63.8% with a custom agent setup.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "mmmu", "value": 81.7, "config": "single attempt (pass@1), Experimental 03-25", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Pro-Model-Card.pdf", "title": "Gemini 2.5 Pro Model Card", "quote": "MMMU single attempt (pass@1) 82.0% 79.6% 81.7% 82.9% 81.6% 74.4% 76.5% 76.0%", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "livecodebench", "value": 70.4, "config": "10/1/2024–2/1/2025 window, single attempt, Experimental 03-25", "reported_by": "official", "source": { "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Pro-Model-Card.pdf", "title": "Gemini 2.5 Pro Model Card", "quote": "Code generation LiveCodeBench UI: 10/1/2024 - 2/1/2025 single attempt (pass@1) — 75.6% 70.4%", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "lmarena-text", "value": 1446, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "75 64 85 gemini-2.5-pro Google · Proprietary 1446 ±2 122,612", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Launched as an 'Experimental (03-25)' model in Google AI Studio and Gemini Advanced; Vertex AI access and pricing followed weeks later. Google's official model card records this exact checkpoint under the column header 'Gemini 2.5 Pro (Experimental 03-25)', distinct from the later 'Preview 05-06' and GA revisions of the same model line (which score differently); the scores recorded here use that Experimental 03-25 column to match this release's launch date." }, { "id": "deepseek-v3-0324", "lab": "deepseek", "name": "DeepSeek-V3-0324", "family": "DeepSeek-V", "date": "2025-03-25", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/raw/main/README.md", "title": "DeepSeek-V3-0324", "quote": "DeepSeek-V3-0324 demonstrates notable improvements over its predecessor, DeepSeek-V3, in several key aspects.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news250325/", "title": "DeepSeek-V3-0324 Release", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu-pro", "value": 81.2, "config": "as reported in the 0324 release notes", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/raw/main/README.md", "title": "DeepSeek-V3-0324 model card", "quote": "MMLU-Pro: 75.9 → 81.2 (+5.3)", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 68.4, "config": "labeled \"GPQA\" in the release notes (variant not spelled out)", "note": "The release notes just say \"GPQA\"; the baseline value cited (59.1) matches DeepSeek-V3's official GPQA-Diamond score, so this is recorded as GPQA Diamond.", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/raw/main/README.md", "title": "DeepSeek-V3-0324 model card", "quote": "GPQA: 59.1 → 68.4 (+9.3)", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "livecodebench", "value": 49.2, "config": "as reported in the 0324 release notes", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324/raw/main/README.md", "title": "DeepSeek-V3-0324 model card", "quote": "LiveCodeBench: 39.2 → 49.2 (+10.0)", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1396, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "151 140 167 deepseek-v3-0324 DeepSeek · MIT 1396 ±4 44,795", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Point update to DeepSeek-V3's weights (same architecture/size); released under the MIT license." }, { "id": "meta-llama-4-behemoth", "lab": "meta", "name": "Llama 4 Behemoth", "family": "Llama", "date": "2025-04-05", "date_precision": "day", "status": "cancelled", "announcement": { "url": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "title": "The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation", "quote": "We also trained a teacher model, Llama 4 Behemoth, that outperforms GPT-4.5, Claude Sonnet 3.7, and Gemini 2.0 Pro on STEM-focused benchmarks such as MATH-500 and GPQA Diamond.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" }, "sources": [ { "url": "https://www.axios.com/2025/05/15/meta-behemoth-llama-scaling-delays", "title": "Meta delays \"Behemoth\" AI model release: report", "retrieved_at": "2026-09-07T03:30:00Z" }, { "url": "https://techcrunch.com/2026/04/08/meta-debuts-the-muse-spark-model-in-a-ground-up-overhaul-of-its-ai/", "title": "Meta debuts the Muse Spark model in a 'ground-up overhaul' of its AI", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [], "notes": "Previewed April 5, 2025 (~288B active / ~2T total params, still training) as the teacher model for Llama 4 Scout/Maverick, with no numeric scores published by Meta. Repeatedly delayed through 2025 (reported mid-training MoE-routing and chunked-attention issues); Meta never issued a formal cancellation, but by April 2026 had moved its frontier effort to the closed-weight Muse Spark under Meta Superintelligence Labs and Behemoth was never publicly released. Recorded here as cancelled/abandoned rather than released." }, { "id": "meta-llama-4-maverick", "lab": "meta", "name": "Llama 4 Maverick", "family": "Llama", "date": "2025-04-05", "date_precision": "day", "status": "released", "announcement": { "url": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "title": "The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation", "quote": "We're introducing Llama 4 Scout and Llama 4 Maverick, the first open-weight natively multimodal models with unprecedented context length support and our first built using a mixture-of-experts (MoE) architecture.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "mmlu", "value": 85.5, "config": "5-shot, macro_avg/acc_char, pre-trained base model", "note": "This is the base/pre-trained model's MMLU; the instruction-tuned model card reports MMLU-Pro instead of plain MMLU.", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md", "title": "Llama 4 Model Card (Meta)", "quote": "Reasoning & Knowledge | MMLU | 5 | macro_avg/acc_char | 79.3 | 85.2 | 79.6 | 85.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "mmlu-pro", "value": 80.5, "config": "0-shot, macro_avg/acc, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md", "title": "Llama 4 Model Card (Meta)", "quote": "Reasoning & Knowledge | MMLU Pro | 0 | macro_avg/acc | 68.9 | 73.4 | 74.3 | 80.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "gpqa-diamond", "value": 69.8, "config": "0-shot, accuracy, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md", "title": "Llama 4 Model Card (Meta)", "quote": "GPQA Diamond | 0 | accuracy | 50.5 | 49.0 | 57.2 | 69.8", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "mmmu", "value": 73.4, "config": "0-shot, accuracy, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md", "title": "Llama 4 Model Card (Meta)", "quote": "Image Reasoning | MMMU | 0 | accuracy | No multimodal support | 69.4 | 73.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } }, { "benchmark": "livecodebench", "value": 43.4, "config": "0-shot, pass@1, window 10/01/2024-02/01/2025, Instruct model", "reported_by": "official", "source": { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md", "title": "Llama 4 Model Card (Meta)", "quote": "Code | LiveCodeBench\n(10/01/2024-02/01/2025) | 0 | pass@1 | 33.3 | 27.7 | 32.8 | 43.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z" } } ], "notes": "Llama 4 Scout (17B active, 16 experts) launched the same day but is a smaller tier; Maverick (17B active, 128 experts) is the flagship of the initial Llama 4 launch. A larger teacher model, Llama 4 Behemoth, was previewed but not released (see meta-llama-4-behemoth)." }, { "id": "openai-o3", "lab": "openai", "name": "OpenAI o3", "family": "o-series", "date": "2025-04-16", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/introducing-o3-and-o4-mini/", "title": "Introducing OpenAI o3 and o4-mini", "quote": "ChatGPT Plus, Pro, and Team users will see o3, o4-mini, and o4-mini-high in the model selector starting today, replacing o1, o3‑mini, and o3‑mini‑high.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "gpqa-diamond", "value": 83.3, "config": "no tools", "note": "Value read from an embedded Vega-Lite chart specification (\"GPQA Diamond\") in the page's raw HTML.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-o3-and-o4-mini/", "quote": "{\"model\":\"o3 (no tools)\",\"value\":83.3,\"labelValue\":83.3,\"color\":\"theme3_yellow\",\"top\":true}", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "aime", "value": 88.9, "config": "AIME 2025, no tools", "note": "Value read from an embedded Vega-Lite chart specification (\"AIME 2025\") in the page's raw HTML.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-o3-and-o4-mini/", "quote": "{\"model\":\"o3 (no tools)\",\"value\":88.9,\"labelValue\":88.9,\"color\":\"theme3_yellow\",\"top\":true}", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "swe-bench-verified", "value": 69.1, "config": "n=477 verified subset, no custom scaffold", "note": "Value read from an embedded Vega-Lite chart specification (\"SWE-Bench Verified (n=477)\") in the page's raw HTML.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-o3-and-o4-mini/", "quote": "{\"model\":\"o3\",\"value\":69.1,\"labelValue\":69.1,\"color\":\"theme3_yellow\",\"top\":true}", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 82.9, "config": "validation split", "note": "Value read from an embedded Vega-Lite chart specification (\"MMMU\") in the page's raw HTML.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-o3-and-o4-mini/", "quote": "{\"model\":\"o3\",\"value\":82.9,\"labelValue\":82.9,\"color\":\"theme3_yellow\",\"top\":true}", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hle", "value": 20.32, "config": "no tools", "note": "Value read from an embedded Vega-Lite chart specification (\"Humanity's Last Exam\") in the page's raw HTML.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-o3-and-o4-mini/", "quote": "{\"model\":\"o3 (no tools)\",\"value\":20.32,\"labelValue\":20.32,\"color\":\"theme3_yellow\",\"top\":true}", "retrieved_at": "2026-09-07T03:30:00Z", "verified": false, "verified_at": "2026-09-07T09:33:07Z" } } ], "notes": "The launch post's benchmark charts are rendered client-side from Vega-Lite JSON specifications embedded in the page's React payload; scores above were read from that embedded JSON (the page's own served content), not from an image." }, { "id": "alibaba-qwen3-235b-a22b", "lab": "alibaba", "name": "Qwen3-235B-A22B", "family": "Qwen", "date": "2025-04-29", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwenlm.github.io/blog/qwen3/", "title": "Qwen3: Think Deeper, Act Faster", "quote": "Today, we are excited to announce the release of Qwen3, the latest addition to the Qwen family of large language models. Our flagship model, Qwen3-235B-A22B, achieves competitive results in benchmark evaluations of coding, math, general capabilities, etc.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" }, "scores": [ { "benchmark": "aime", "value": 81.5, "config": "AIME 2025, thinking mode", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2505.09388v1", "title": "Qwen3 Technical Report", "quote": "the flagship model Qwen3-235B-A22B achieves 85.7 on AIME’24 and 81.5 on AIME’25 ( AIME, 2025 ) , 70.7 on LiveCodeBench v5 ( Jain et al., 2024 ) , 2,056 on CodeForces, and 70.8 on BFCL v3 ( Yan et al., 2024 ) .", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "livecodebench", "value": 70.7, "config": "LiveCodeBench v5, thinking mode", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2505.09388v1", "title": "Qwen3 Technical Report", "quote": "the flagship model Qwen3-235B-A22B achieves 85.7 on AIME’24 and 81.5 on AIME’25 ( AIME, 2025 ) , 70.7 on LiveCodeBench v5 ( Jain et al., 2024 ) , 2,056 on CodeForces, and 70.8 on BFCL v3 ( Yan et al., 2024 ) .", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } }, { "benchmark": "gpqa-diamond", "value": 71.1, "config": "thinking mode, avg@10", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2505.09388v1", "title": "Qwen3 Technical Report, Table 11", "quote": "GPQA-Diamond 78.0 71.5 80.2 84.0 71.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Table 11 (Qwen3-235B-A22B Thinking vs. reasoning baselines) columns: OpenAI-o1, DeepSeek-R1, Grok-3-Beta (Think), Gemini2.5-Pro, Qwen3-235B-A22B." } }, { "benchmark": "lmarena-text", "value": 1375, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "176 167 184 qwen3-235b-a22b Alibaba · Apache 2.0 1375 ±5 25,864", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "mistral-mistral-medium-3", "lab": "mistral", "name": "Mistral Medium 3", "family": "Mistral Large", "date": "2025-05-07", "date_precision": "day", "status": "released", "announcement": { "url": "https://mistral.ai/news/mistral-medium-3/", "title": "Medium is the new large.", "quote": "And today, we are excited to announce Mistral Medium 3, pushing efficiency and usability of language models even further.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" }, "scores": [ { "benchmark": "aime", "value": 21.2, "config": "AIME 2025, pass@1", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "AIME'25 21.2 / 30.0 64.9 / 83.3 28.8 - 70.0", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'AIME'25', shown as pass@1 / maj@64. Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1. Value used here is Mistral Medium 3's pass@1 (21.2)." } }, { "benchmark": "math", "value": 91, "config": "MATH-500", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "MATH-500 91.0 94.3 90.2 95.9 97.3", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'MATH-500'. Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } }, { "benchmark": "gpqa-diamond", "value": 59.6, "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "GPQA 59.6 70.8 59.1 73.3 71.5", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'GPQA'; Diamond subset not explicitly specified. Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } }, { "benchmark": "livecodebench", "value": 30, "config": "v6", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "LiveCodeBench (v6) 30.0 50.3 - - -", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'LiveCodeBench (v6)'. Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } }, { "benchmark": "hle", "value": 4.4, "config": "text-only", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "Humanity's Last Exam 4.4 9.0 - - 8.6", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'Humanity's Last Exam', text subset only (per Table 2 caption). Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } } ], "notes": "Included as the period's flagship because Mistral Large was not refreshed between Large 2.1 (Nov 2024) and Large 3 (Dec 2025); Medium 3 was Mistral's highest publicly available tier in this window. Scores are baseline numbers for Mistral Medium 3 as reported in Mistral's own Magistral technical report (arXiv:2506.10910), since the Mistral Medium 3 launch post itself renders all benchmark tables as images." }, { "id": "anthropic-claude-opus-4", "lab": "anthropic", "name": "Claude Opus 4", "family": "Claude Opus", "date": "2025-05-22", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-4", "title": "Introducing Claude 4", "quote": "Both models are available on our API, Amazon Bedrock, and Google Cloud's Vertex AI.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "scores": [ { "benchmark": "swe-bench-verified", "value": 72.5, "config": "pass@1, no parallel test-time compute, Claude Code as agent framework", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-4", "quote": "leading on SWE-bench (72.5%) and Terminal-bench (43.2%)", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "gpqa-diamond", "value": 79.6, "config": "no parallel test-time compute", "note": "Table gives '79.6% / 83.3%'; the second figure benefits from parallel test-time compute (best-of-N via an internal scoring model), per the page's methodology footnote 5.", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-4", "quote": "79.6% / 83.3%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "mmmu", "value": 76.5, "config": "validation split", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-4", "quote": "76.5%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "MMMU (validation) row of the benchmark table; Opus 4's column value." } }, { "benchmark": "aime", "value": 75.5, "config": "AIME 2025, no parallel test-time compute", "note": "Table gives '75.5% / 90.0%'; the second figure benefits from parallel test-time compute per methodology footnote 5.", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-4", "quote": "75.5% / 90.0%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "lmarena-text", "value": 1426, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "107 91 125 claude-opus-4-20250514-thinking-16k Anthropic · Proprietary 1426 ±4 35,743", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Launched alongside Claude Sonnet 4 at the 'Code with Claude' developer event; the API model snapshot id (claude-opus-4-20250514) carries an earlier internal date than the May 22, 2025 public launch, which is used here. TAU-bench figures reported on this page are for the original tau-bench, not the tau2-bench in this project's basket, so they are not recorded as tau2-bench scores." }, { "id": "deepseek-r1-0528", "lab": "deepseek", "name": "DeepSeek-R1-0528", "family": "DeepSeek-R", "date": "2025-05-28", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/README.md", "title": "DeepSeek-R1-0528", "quote": "The DeepSeek R1 model has undergone a minor version upgrade, with the current version being DeepSeek-R1-0528.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news250528/", "title": "DeepSeek-R1-0528 Release", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu-pro", "value": 85, "config": "EM", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/README.md", "title": "DeepSeek-R1-0528 model card", "quote": "| | MMLU-Pro (EM) | 84.0 | 85.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 81, "config": "pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/README.md", "title": "DeepSeek-R1-0528 model card", "quote": "| | GPQA-Diamond (Pass@1) | 71.5 | 81.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hle", "value": 17.7, "config": "pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/README.md", "title": "DeepSeek-R1-0528 model card", "quote": "| | Humanity's Last Exam (Pass@1) | 8.5 | 17.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "swe-bench-verified", "value": 57.6, "config": "Resolved", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/README.md", "title": "DeepSeek-R1-0528 model card", "quote": "| | SWE Verified (Resolved) | 49.2 | 57.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "aime", "value": 87.5, "config": "AIME 2025, pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528/raw/main/README.md", "title": "DeepSeek-R1-0528 model card", "quote": "| | AIME 2025 (Pass@1) | 70.0 | 87.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1421, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "115 97 134 deepseek-r1-0528 DeepSeek · MIT 1421 ±6 18,092", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Minor version upgrade of DeepSeek-R1 (same architecture/size), with increased post-training compute." }, { "id": "mistral-magistral-medium", "lab": "mistral", "name": "Magistral Medium", "family": "Magistral", "date": "2025-06-10", "date_precision": "day", "status": "released", "announcement": { "url": "https://mistral.ai/news/magistral/", "title": "Magistral", "quote": "Today, we're excited to announce our latest contribution to AI research with Magistral — our first reasoning model.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" }, "scores": [ { "benchmark": "aime", "value": 73.6, "config": "AIME 2024, pass@1", "reported_by": "official", "source": { "url": "https://mistral.ai/news/magistral/", "title": "Magistral", "quote": "Magistral Medium scored 73.6% on AIME2024, and 90% with majority voting @64.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" } }, { "benchmark": "math", "value": 94.3, "config": "MATH-500", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "MATH-500 91.0 94.3 90.2 95.9 97.3", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'MATH-500'. Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } }, { "benchmark": "gpqa-diamond", "value": 70.8, "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "GPQA 59.6 70.8 59.1 73.3 71.5", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'GPQA'; Diamond subset not explicitly specified. Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } }, { "benchmark": "livecodebench", "value": 50.3, "config": "v6", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "LiveCodeBench (v6) 30.0 50.3 - - -", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'LiveCodeBench (v6)'. Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } }, { "benchmark": "hle", "value": 9, "config": "text-only", "reported_by": "official", "source": { "url": "https://arxiv.org/abs/2506.10910", "title": "Magistral (Mistral AI technical report) — Table 2", "quote": "Humanity's Last Exam 4.4 9.0 - - 8.6", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:30:07Z", "note": "Table 2 row 'Humanity's Last Exam', text subset only (per Table 2 caption). Column order: Mistral Medium 3, Magistral Medium, DeepSeek-V3, DeepSeek-R1-Zero, DeepSeek-R1." } } ], "notes": "Reasoning-specialised flagship (family 'Magistral'), trained with RL on top of Mistral Medium 3 and released alongside the open-weight Magistral Small; included per the brief as the reasoning-flagship entry for this period." }, { "id": "xai-grok-4", "lab": "xai", "name": "Grok 4", "family": "Grok", "date": "2025-07-09", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok-4", "title": "Grok 4", "quote": "Grok 4 is the most intelligent model in the world. It includes native tool use and real-time search integration, and is available now to SuperGrok and Premium+ subscribers, as well as through the xAI API.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z", "note": "Quote is xAI's own summary of the grok-4 launch article as shown on the https://x.ai/news index; the dedicated article itself does not render a hero paragraph as extractable text." }, "scores": [ { "benchmark": "arc-agi-2", "value": 15.9, "reported_by": "official", "source": { "url": "https://x.ai/news/grok-4", "title": "Grok 4", "quote": "setting a new state-of-the-art for closed models on ARC-AGI V2 with 15.9% (nearly double Opus's ~8.6%, +8pp over previous high).", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "gpqa-diamond", "value": 87.5, "config": "no tools", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-4", "title": "Grok 4", "quote": "\\\"group\\\":\\\"GPQA\\\",\\\"tool\\\":\\\"Python\\\",\\\"data\\\":[{\\\"model\\\":\\\"O3_NT\\\",\\\"score\\\":83.3,\\\"tools\\\":false},{\\\"model\\\":\\\"GEMINI_25_PRO_NT\\\",\\\"score\\\":86.4,\\\"tools\\\":false},{\\\"model\\\":\\\"CLAUDE_OPUS_4_NT\\\",\\\"score\\\":79.6,\\\"tools\\\":false},{\\\"model\\\":\\\"GROK_4_NT\\\",\\\"score\\\":87.5,\\\"tools\\\":false}", "retrieved_at": "2026-09-07T04:00:19Z", "verified": false, "verified_at": "2026-09-07T09:30:58Z", "note": "value embedded in page chart payload" } }, { "benchmark": "aime", "value": 91.7, "config": "AIME 2025, no tools", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-4", "title": "Grok 4", "quote": "\\\"group\\\":\\\"AIME’25\\\",\\\"tool\\\":\\\"Python\\\",\\\"data\\\":[{\\\"model\\\":\\\"O3_NT\\\",\\\"score\\\":88.9,\\\"tools\\\":false},{\\\"model\\\":\\\"GEMINI_25_PRO_NT\\\",\\\"score\\\":88,\\\"tools\\\":false},{\\\"model\\\":\\\"CLAUDE_OPUS_4_NT\\\",\\\"score\\\":75.5,\\\"tools\\\":false},{\\\"model\\\":\\\"GROK_4_NT\\\",\\\"score\\\":91.7,\\\"tools\\\":false}", "retrieved_at": "2026-09-07T04:00:19Z", "verified": false, "verified_at": "2026-09-07T09:30:58Z", "note": "value embedded in page chart payload" } }, { "benchmark": "livecodebench", "value": 79, "config": "Jan-May 2025 window, no tools", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-4", "title": "Grok 4", "quote": "\\\"group\\\":\\\"LiveCodeBench (Jan - May)\\\",\\\"tool\\\":\\\"Python\\\",\\\"data\\\":[{\\\"model\\\":\\\"O3_NT\\\",\\\"score\\\":72,\\\"tools\\\":false},{\\\"model\\\":\\\"GEMINI_25_PRO_NT\\\",\\\"score\\\":74.2,\\\"tools\\\":false},{\\\"model\\\":\\\"GROK_4_NT\\\",\\\"score\\\":79,\\\"tools\\\":false}", "retrieved_at": "2026-09-07T04:00:19Z", "verified": false, "verified_at": "2026-09-07T09:30:58Z", "note": "value embedded in page chart payload" } }, { "benchmark": "hle", "value": 25.4, "config": "no tools, text+multimodal", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-4", "title": "Grok 4", "quote": "\\\"model\\\":\\\"O3\\\",\\\"score\\\":24.9,\\\"tools\\\":true},{\\\"model\\\":\\\"GROK_4_NT\\\",\\\"score\\\":25.4,\\\"tools\\\":false}", "retrieved_at": "2026-09-07T04:00:19Z", "verified": false, "verified_at": "2026-09-07T09:30:58Z", "note": "value embedded in page chart payload" } } ], "notes": "Released Jul 9, 2025 to SuperGrok/X Premium+ subscribers and the xAI API, alongside a heavier 'Grok 4 Heavy' parallel-test-time-compute variant (excluded here as a non-flagship tier). GPQA Diamond (87.5%, no tools), AIME 2025 (91.7% no tools / 98.8% with a Python tool — no-tools recorded per basket preferred_config), LiveCodeBench (79%, Jan-May 2025 window, no tools) and HLE (25.4%, no tools) are not in the page's visible prose but are present as literal JSON inside the page's React/Next.js payload (`self.__next_f.push` chart data), re-fetched 2026-09-07 and quoted verbatim from that payload per source.note; ARC-AGI-2 is stated directly in the article's own prose." }, { "id": "moonshot-kimi-k2", "lab": "moonshot", "name": "Kimi K2", "family": "Kimi K", "date": "2025-07-11", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "Kimi K2 is a state-of-the-art mixture-of-experts (MoE) language model with 32 billion activated parameters and 1 trillion total parameters.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Date is the Hugging Face repo creation date for Kimi-K2-Instruct; the README carries no separate dated announcement sentence." }, "scores": [ { "benchmark": "livecodebench", "value": 53.7, "config": "LiveCodeBench v6 (Aug24-May25), pass@1", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "LiveCodeBench v6 (Aug 24 - May 25) Pass@1 53.7 46.9 37.0 48.5 47.4 44.7 44.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Table columns: Kimi K2 Instruct, DeepSeek-V3-0324, Qwen3-235B-A22B (non-thinking), Claude Sonnet 4, Claude Opus 4, GPT-4.1, Gemini 2.5 Flash Preview." } }, { "benchmark": "swe-bench-verified", "value": 65.8, "config": "Agentic Coding, single attempt", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "SWE-bench Verified (Agentless Coding) Single Patch w/o Test (Acc) 51.8 36.6 39.4 50.2 53.0 40.8 32.6 SWE-bench Verified (Agentic Coding) Single Attempt (Acc) 65.8 38.8", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "tau2-bench", "value": 64.3, "config": "avg@4", "note": "Lab does not publish a combined figure; this is the unweighted mean of retail (70.6), airline (56.5), and telecom (65.8), each avg@4, per this project's methodology for domain-split Tau2-bench scores.", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "Tau2 retail Avg@4 70.6 69.1 57.0 75.0 81.8 74.8 64.3 Tau2 airline Avg@4 56.5 39.0 26.5 55.5 60.0 54.5 42.5 Tau2 telecom Avg@4 65.8 32.5 22.1 45.2 57.0 38.6 16.9", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "aime", "value": 49.5, "config": "AIME 2025, avg@64", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "AIME 2025 Avg@64 49.5 46.7 24.7 * 33.1 * 33.9 * 37.0 46.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "gpqa-diamond", "value": 75.1, "config": "avg@8", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "GPQA-Diamond Avg@8 75.1 68.4 * 62.9 * 70.0 * 74.9 * 66.3 68.2", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "mmlu", "value": 89.5, "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "MMLU EM 89.5 89.4 87.0 91.5 92.9 90.4 90.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "mmlu-pro", "value": 81.1, "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "MMLU-Pro EM 81.1 81.2 * 77.3 83.7 86.6 81.8 79.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "hle", "value": 4.7, "config": "text-only, no tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "title": "moonshotai/Kimi-K2-Instruct model card", "quote": "Humanity's Last Exam (Text Only) - 4.7 5.2 5.7 5.8 7.1 3.7 5.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } } ] }, { "id": "zhipu-glm-4.5", "lab": "zhipu", "name": "GLM-4.5", "family": "GLM", "date": "2025-07-28", "date_precision": "day", "status": "released", "announcement": { "url": "http://web.archive.org/web/20250801163650/https://z.ai/blog/glm-4.5", "title": "GLM-4.5: Reasoning, Coding, and Agentic Abilities", "quote": "Today, we introduce two new GLM family members: GLM-4.5 and GLM-4.5-Air — our latest flagship models.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "z.ai/blog/glm-4.5 currently mis-renders unrelated cached content; Wayback Machine snapshot from shortly after launch used instead." }, "scores": [ { "benchmark": "mmlu-pro", "value": 84.6, "config": "thinking mode", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20250801163650/https://z.ai/blog/glm-4.5", "title": "GLM-4.5: Reasoning, Coding, and Agentic Abilities", "quote": "MMLU Pro 84.6 81.4 85.3 87.3 86.2 84.9 84.5 86.6", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Row order: GLM-4.5, GLM-4.5-Air, o3, Claude 4 Opus, Gemini 2.5 Pro, DeepSeek-R1-0528, Qwen3-235B-Thinking 2507, Grok 4." } }, { "benchmark": "aime", "value": 91, "config": "AIME 2024, avg@32", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20250801163650/https://z.ai/blog/glm-4.5", "title": "GLM-4.5: Reasoning, Coding, and Agentic Abilities", "quote": "AIME24 91.0 89.4 90.3 75.7 88.7 89.3 94.1 94.3", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Same row order as MMLU-Pro." } }, { "benchmark": "gpqa-diamond", "value": 79.1, "config": "avg@8", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20250801163650/https://z.ai/blog/glm-4.5", "title": "GLM-4.5: Reasoning, Coding, and Agentic Abilities", "quote": "GPQA 79.1 75.0 82.7 79.6 84.4 81.3 81.1 87.7", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Same row order as MMLU-Pro; reported as 'GPQA' (avg@8), Diamond subset not explicitly specified." } }, { "benchmark": "hle", "value": 14.4, "config": "text-only", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20250801163650/https://z.ai/blog/glm-4.5", "title": "GLM-4.5: Reasoning, Coding, and Agentic Abilities", "quote": "HLE 14.4 10.6 20.0 11.7 21.1 14.9 15.8 23.9", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "'For the HLE benchmark, only the text-based questions were evaluated, with correctness judged by gpt-4o.'" } }, { "benchmark": "livecodebench", "value": 72.9, "config": "2407-2501 window", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20250801163650/https://z.ai/blog/glm-4.5", "title": "GLM-4.5: Reasoning, Coding, and Agentic Abilities", "quote": "LiveCodeBench (2407-2501) 72.9 70.7 78.4 63.6 80.1 77.0 78.2 81.9", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "z.ai/blog/glm-4.5 currently mis-renders unrelated cached content; Wayback Machine snapshot from shortly after launch used instead." } }, { "benchmark": "swe-bench-verified", "value": 64.2, "config": "OpenHands v0.34.0, 100 iterations, temperature=0.6", "reported_by": "official", "source": { "url": "http://web.archive.org/web/20250801163650/https://z.ai/blog/glm-4.5", "title": "GLM-4.5: Reasoning, Coding, and Agentic Abilities", "quote": "SWE-bench Verified 1 64.2 57.6 69.1 48.6 67.8 70.4 49.0 41.4 65.4", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "note": "Row order: GLM-4.5, GLM-4.5-Air, o3, GPT-4.1, Claude 4 Opus, Claude 4 Sonnet, Gemini 2.5 Pro, DeepSeek-R1-0528, Kimi K2." } }, { "benchmark": "lmarena-text", "value": 1411, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "134 113 146 glm-4.5 Z.ai · MIT 1411 ±5 23,715", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "anthropic-claude-opus-4.1", "lab": "anthropic", "name": "Claude Opus 4.1", "family": "Claude Opus", "date": "2025-08-05", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-opus-4-1", "title": "Claude Opus 4.1", "quote": "Opus 4.1 is now available to paid Claude users and in Claude Code.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "scores": [ { "benchmark": "swe-bench-verified", "value": 74.5, "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-opus-4-1", "quote": "Opus 4.1 advances our state-of-the-art coding performance to 74.5% on SWE-bench Verified.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "lmarena-text", "value": 1450, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "68 55 80 claude-opus-4-1-20250805-thinking-16k Anthropic · Proprietary 1450 ±3 48,828", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Date confirmed by the API model id claude-opus-4-1-20250805. A broader benchmark table comparing Opus 4.1 to prior models is shown only as an image on the launch page; no other text-extractable scores were found within budget." }, { "id": "openai-gpt-5", "lab": "openai", "name": "GPT-5", "family": "GPT", "date": "2025-08-07", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/introducing-gpt-5/", "title": "Introducing GPT-5", "quote": "GPT‑5 is available to all users, with Plus subscribers getting more usage, and Pro subscribers getting access to GPT‑5 pro, a version with extended reasoning for even more comprehensive and accurate answers.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "aime", "value": 94.6, "config": "AIME 2025, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5/", "quote": "It sets a new state of the art across math (94.6% on AIME 2025 without tools), real-world coding (74.9% on SWE-bench Verified, 88% on Aider Polyglot), multimodal understanding (84.2% on MMMU), and health (46.2% on HealthBench Hard)", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "swe-bench-verified", "value": 74.9, "config": "n=477 verified subset", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5/", "quote": "It sets a new state of the art across math (94.6% on AIME 2025 without tools), real-world coding (74.9% on SWE-bench Verified, 88% on Aider Polyglot), multimodal understanding (84.2% on MMMU), and health (46.2% on HealthBench Hard)", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 84.2, "config": "validation split", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5/", "quote": "It sets a new state of the art across math (94.6% on AIME 2025 without tools), real-world coding (74.9% on SWE-bench Verified, 88% on Aider Polyglot), multimodal understanding (84.2% on MMMU), and health (46.2% on HealthBench Hard)", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gpqa-diamond", "value": 88.4, "config": "GPT-5 pro, extended reasoning, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5/", "quote": "With GPT‑5 pro's extended reasoning, the model also sets a new SOTA on GPQA, scoring 88.4% without tools.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1434, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "93 79 109 gpt-5-high OpenAI · Proprietary 1434 ±5 31,433", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "GPT-5 launched as a unified system (Instant/Thinking/Pro routed automatically); GPQA Diamond figure of 88.4% is specifically GPT-5 pro's extended-reasoning score, the highest in the family at launch. Launch date corroborated by extensive same-day press coverage (August 7, 2025)." }, { "id": "deepseek-v3.1", "lab": "deepseek", "name": "DeepSeek-V3.1", "family": "DeepSeek-V", "date": "2025-08-21", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/raw/main/README.md", "title": "DeepSeek-V3.1", "quote": "DeepSeek-V3.1 is a hybrid model that supports both thinking mode and non-thinking mode.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news250821/", "title": "DeepSeek-V3.1 Release", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu-pro", "value": 84.8, "config": "EM, thinking mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/raw/main/README.md", "title": "DeepSeek-V3.1 model card", "quote": "| | MMLU-Pro (EM) | 83.7 | 81.2 | 84.8 | 85.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 80.1, "config": "pass@1, thinking mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/raw/main/README.md", "title": "DeepSeek-V3.1 model card", "quote": "| | GPQA-Diamond (Pass@1) | 74.9 | 68.4 | 80.1 | 81.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hle", "value": 15.9, "config": "pass@1, thinking mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/raw/main/README.md", "title": "DeepSeek-V3.1 model card", "quote": "| | Humanity's Last Exam (Pass@1) | - | - | 15.9 | 17.7", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "livecodebench", "value": 74.8, "config": "window 2408-2505, pass@1, thinking mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/raw/main/README.md", "title": "DeepSeek-V3.1 model card", "quote": "| | LiveCodeBench (2408-2505) (Pass@1) | 56.4 | 43.0 | 74.8 | 73.3", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "aime", "value": 88.4, "config": "AIME 2025, pass@1, thinking mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1/raw/main/README.md", "title": "DeepSeek-V3.1 model card", "quote": "| | AIME 2025 (Pass@1) | 49.8 | 51.3 | 88.4 | 87.5", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1417, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "121 102 139 deepseek-v3.1 DeepSeek · MIT 1417 ±6 14,609", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "First hybrid think/non-think DeepSeek-V model; scores recorded here are the thinking-mode configuration." }, { "id": "alibaba-qwen3-max", "lab": "alibaba", "name": "Qwen3-Max", "family": "Qwen", "date": "2025-09-24", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwen.ai/blog?id=qwen3-max", "title": "Qwen3-Max: Just Scale it", "quote": "Following the release of the Qwen3-2507 series, we are thrilled to introduce Qwen3-Max — our largest and most capable model to date.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "via": "r.jina.ai", "note": "The blog page carries no visible publish date. Qwen3-Max-Preview went up Sep 5, 2025; contemporaneous press (Dataconomy, MarkTechPost) place this post's official/GA release on Sep 24, 2025, distinct from the preview." }, "scores": [ { "benchmark": "swe-bench-verified", "value": 69.6, "config": "Instruct", "reported_by": "official", "source": { "url": "https://qwen.ai/blog?id=qwen3-max", "title": "Qwen3-Max: Just Scale it", "quote": "On SWE-Bench Verified, a benchmark focused on solving real-world coding challenges, Qwen3-Max-Instruct achieves an impressive score of 69.6, placing it firmly among the world's top-performing models.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "via": "r.jina.ai" } }, { "benchmark": "tau2-bench", "value": 74.8, "config": "Instruct", "reported_by": "official", "source": { "url": "https://qwen.ai/blog?id=qwen3-max", "title": "Qwen3-Max: Just Scale it", "quote": "on Tau2-Bench — a rigorous evaluation of agent tool-calling proficiency — Qwen3-Max-Instruct delivers a breakthrough score of 74.8, surpassing both Claude Opus 4 and DeepSeek V3.1.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "via": "r.jina.ai" } } ] }, { "id": "deepseek-v3.2-exp", "lab": "deepseek", "name": "DeepSeek-V3.2-Exp", "family": "DeepSeek-V", "date": "2025-09-29", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/raw/main/README.md", "title": "DeepSeek-V3.2-Exp", "quote": "We are excited to announce the official release of DeepSeek-V3.2-Exp, an experimental version of our model.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news250929/", "title": "Introducing DeepSeek-V3.2-Exp", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu-pro", "value": 85, "config": "reasoning mode w/o tool use", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/raw/main/README.md", "title": "DeepSeek-V3.2-Exp model card", "quote": "| MMLU-Pro | 85.0 | 85.0 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 79.9, "config": "reasoning mode w/o tool use", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/raw/main/README.md", "title": "DeepSeek-V3.2-Exp model card", "quote": "| GPQA-Diamond | 80.7 | 79.9 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hle", "value": 19.8, "config": "reasoning mode w/o tool use", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/raw/main/README.md", "title": "DeepSeek-V3.2-Exp model card", "quote": "| Humanity's Last Exam | 21.7 | 19.8 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "livecodebench", "value": 74.1, "config": "reasoning mode w/o tool use", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/raw/main/README.md", "title": "DeepSeek-V3.2-Exp model card", "quote": "| LiveCodeBench | 74.9 | 74.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "aime", "value": 89.3, "config": "AIME 2025, reasoning mode w/o tool use", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/raw/main/README.md", "title": "DeepSeek-V3.2-Exp model card", "quote": "| AIME 2025 | 88.4 | 89.3 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "swe-bench-verified", "value": 67.8, "config": "agentic tool use", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/raw/main/README.md", "title": "DeepSeek-V3.2-Exp model card", "quote": "| SWE Verified | 68.4 | 67.8 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1422, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "114 94 132 deepseek-v3.2-exp DeepSeek · MIT 1422 ±6 11,756", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Experimental release introducing DeepSeek Sparse Attention (DSA); DeepSeek deliberately aligned its training configuration with V3.1-Terminus to isolate the effect of sparse attention, so scores are close to V3.1's." }, { "id": "zhipu-glm-4.6", "lab": "zhipu", "name": "GLM-4.6", "family": "GLM", "date": "2025-09-30", "date_precision": "day", "status": "released", "announcement": { "url": "https://z.ai/blog/glm-4.6", "title": "GLM-4.6: Advanced Agentic, Reasoning and Coding Capabilities", "quote": "Today, we are releasing the latest version of our flagship model: GLM-4.6.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "mmlu-pro", "value": 83.2, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| MMLU-Pro | 84.3 | 83.2 | 84.6 | 85.0 | 90.1 | 88.2 | 87.5 | 87.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "gpqa-diamond", "value": 81, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| GPQA-Diamond | 85.7 | 81.0 | 84.5 | 82.4 | 91.9 | 83.4 | 85.7 | 88.1 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "hle", "value": 17.2, "config": "text-only", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| HLE | 24.8 | 17.2 | 23.9 | 25.1 | 37.5 | 13.7 | 26.3 | 25.7 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "aime", "value": 93.9, "config": "AIME 2025", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| AIME 2025 | 95.7 | 93.9 | 94.5 | 93.1 | 95.0 | 87.0 | 94.6 | 94.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "livecodebench", "value": 82.8, "config": "v6", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| LiveCodeBench-v6 | 84.9 | 82.8 | 83.1 | 83.3 | 90.7 | 64.0 | 87.0 | 87.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "swe-bench-verified", "value": 68, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| SWE-bench Verified | 73.8 | 68.0 | 71.3 | 73.1 | 76.2 | 77.2 | 74.9 | 76.3 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "terminal-bench-2", "value": 24.5, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| Terminal Bench 2.0 | 41.0 | 24.5 | 35.7 | 46.4 | 54.2 | 42.8 | 35.2 | 47.6 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "tau2-bench", "value": 75.2, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| τ²-Bench | 87.4 | 75.2 | 74.3 | 85.3 | 90.7 | 87.2 | 82.4 | 82.7 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "lmarena-text", "value": 1425, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "110 94 125 glm-4.6 Z.ai · MIT 1425 ±4 35,068", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Scores sourced from the later GLM-4.7 launch post's official comparison table (Z.ai's own re-reported numbers for GLM-4.6), since the GLM-4.6 launch post itself renders its benchmark chart as an image only." }, { "id": "moonshot-kimi-k2-thinking", "lab": "moonshot", "name": "Kimi K2 Thinking", "family": "Kimi K", "date": "2025-11-06", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "title": "moonshotai/Kimi-K2-Thinking model card", "quote": "Kimi K2 Thinking is the latest, most capable version of open-source thinking model.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" }, "scores": [ { "benchmark": "mmlu-pro", "value": 84.6, "config": "no tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "title": "moonshotai/Kimi-K2-Thinking model card", "quote": "**MMLU-Pro** | no tools | 84.6 | 87.1 | 87.5 | 81.9 | 85.0 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Table columns: K2 Thinking, GPT-5 (High), Claude Sonnet 4.5 (Thinking), K2 0905, DeepSeek-V3.2." } }, { "benchmark": "aime", "value": 94.5, "config": "AIME25, no tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "title": "moonshotai/Kimi-K2-Thinking model card", "quote": "**AIME25** | no tools | 94.5 | 94.6 | 87.0 | 51.0 | 89.3 | 91.7 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "gpqa-diamond", "value": 84.5, "config": "no tools", "note": "Lab reports this simply as 'GPQA'; recorded as GPQA Diamond per standard practice, though the source does not use the word 'Diamond'.", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "title": "moonshotai/Kimi-K2-Thinking model card", "quote": "**GPQA** | no tools | 84.5 | 85.7 | 83.4 | 74.2 | 79.9 | 87.5 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "hle", "value": 23.9, "config": "text-only, no tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "title": "moonshotai/Kimi-K2-Thinking model card", "quote": "HLE (Text-only)** | no tools | 23.9 | 26.3 | 19.8* | 7.9 | 19.8 | 25.4 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "livecodebench", "value": 83.1, "config": "LiveCodeBenchV6, no tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "title": "moonshotai/Kimi-K2-Thinking model card", "quote": "LiveCodeBenchV6** | no tools | 83.1 | 87.0* | 64.0* | 56.1* | 74.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "swe-bench-verified", "value": 71.3, "config": "w/ tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "title": "moonshotai/Kimi-K2-Thinking model card", "quote": "SWE-bench Verified** | w/ tools | 71.3 | 74.9 | 77.2 | 69.2 | 67.8 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } } ] }, { "id": "openai-gpt-5.1", "lab": "openai", "name": "GPT-5.1", "family": "GPT", "date": "2025-11-12", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/gpt-5-1/", "title": "GPT-5.1: A smarter, more conversational ChatGPT", "quote": "GPT‑5.1 Instant and Thinking begin rolling out today, starting with paid (Pro, Plus, Go, Business) users and then to free and logged-out users.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "sources": [ { "url": "https://openai.com/index/gpt-5-1-for-developers/", "title": "Introducing GPT-5.1 for developers", "retrieved_at": "2026-09-07T03:30:00Z", "via": "r.jina.ai" } ], "scores": [ { "benchmark": "swe-bench-verified", "value": 76.3, "config": "high reasoning effort, all 500 problems", "note": "Appendix table, columns GPT-5.1 (high) / GPT-5 (high).", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-5-1-for-developers/", "quote": "SWE-bench Verified (all 500 problems)76.3%72.8%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gpqa-diamond", "value": 88.1, "config": "high reasoning effort, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-5-1-for-developers/", "quote": "GPQA Diamond (no tools)88.1%85.7%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "aime", "value": 94, "config": "AIME 2025, high reasoning effort, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-5-1-for-developers/", "quote": "AIME 2025(no tools)94.0%94.6%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 85.4, "config": "high reasoning effort", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-5-1-for-developers/", "quote": "MMMU 85.4%84.2%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1439, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "84 74 100 gpt-5.1 OpenAI · Proprietary 1439 ±4 43,008", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Consumer rollout announced on openai.com/index/gpt-5-1/ (November 12, 2025); developer/API scores are from the companion openai.com/index/gpt-5-1-for-developers/ post's evaluation appendix (GPT-5.1 (high) vs GPT-5 (high))." }, { "id": "xai-grok-4.1", "lab": "xai", "name": "Grok 4.1", "family": "Grok", "date": "2025-11-17", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok-4-1", "title": "Grok 4.1", "quote": "Grok 4.1 is now available to all users on grok.com, 𝕏, and the iOS and Android apps.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 1459, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "53 43 66 grok-4.1 SpaceXAI · Proprietary 1459 ±3 66,382", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "A personality/alignment-focused update to Grok 4 (creative writing, emotional intelligence, reduced hallucination rate), not a new pretraining/reasoning generation. Its own launch post reports only LMArena Elo, EQ-Bench3 and FActScore/hallucination-rate figures, none of which are in the tracked benchmark basket, so no scores are recorded for this release." }, { "id": "google-gemini-3-pro", "lab": "google", "name": "Gemini 3 Pro", "family": "Gemini Pro", "date": "2025-11-18", "date_precision": "day", "status": "released", "announcement": { "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3/", "title": "A new era of intelligence with Gemini 3", "quote": "Today we’re taking another big step on the path toward AGI and releasing Gemini 3.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "sources": [ { "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-3-Pro-Model-Card.pdf", "title": "Gemini 3 Pro Model Card", "quote": "Model Release: November 2025", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } ], "scores": [ { "benchmark": "hle", "value": 37.5, "config": "no tools", "reported_by": "official", "source": { "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3/", "title": "A new era of intelligence with Gemini 3", "quote": "It demonstrates PhD-level reasoning with top scores on Humanity’s Last Exam (37.5% without the usage of any tools) and GPQA Diamond (91.9%).", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "gpqa-diamond", "value": 91.9, "reported_by": "official", "source": { "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3/", "title": "A new era of intelligence with Gemini 3", "quote": "It demonstrates PhD-level reasoning with top scores on Humanity’s Last Exam (37.5% without the usage of any tools) and GPQA Diamond (91.9%).", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "mmmu", "value": 81, "config": "MMMU-Pro", "reported_by": "official", "source": { "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3/", "title": "A new era of intelligence with Gemini 3", "quote": "Beyond text, Gemini 3 Pro redefines multimodal reasoning with 81% on MMMU-Pro and 87.6% on Video-MMMU.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "swe-bench-verified", "value": 76.2, "reported_by": "official", "source": { "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3/", "title": "A new era of intelligence with Gemini 3", "quote": "it greatly outperforms 2.5 Pro on SWE-bench Verified (76.2%), a benchmark that measures coding agents.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "terminal-bench-2", "value": 54.2, "reported_by": "official", "source": { "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3/", "title": "A new era of intelligence with Gemini 3", "quote": "It also scores 54.2% on Terminal-Bench 2.0, which tests a model’s tool use ability to operate a computer via terminal", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "arc-agi-2", "value": 31.1, "config": "Thinking (High), ARC Prize Verified", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "ARC-AGI-2 Abstract reasoning puzzlesARC Prize Verified77.1%31.1%", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z", "note": "Base Gemini 3 Pro figure from the current DeepMind model-comparison page, distinct from the higher-effort 'Gemini 3 Deep Think' mode's 45.1% mentioned below." } }, { "benchmark": "lmarena-text", "value": 1486, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "16 7 25 gemini-3-pro Google · Proprietary 1486 ±4 40,668", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Shipped in preview across the Gemini app, AI Mode in Search, AI Studio, Vertex AI and Google Antigravity on day one. The model card's own capability-benchmark table is rendered as an image with no extractable text, so numeric scores here are taken from the launch blog post instead, plus one ARC-AGI-2 figure recovered from the live deepmind.google/models/gemini/pro comparison table (which lists Gemini 3 Pro as a competitor column alongside 3.1 Pro). A more capable 'Gemini 3 Deep Think' mode (HLE 41.0%, GPQA 93.8%, ARC-AGI-2 45.1% with code execution) was announced the same day but rolled out to Google AI Ultra subscribers later; it is treated as a mode of this release rather than a separate flagship." }, { "id": "anthropic-claude-opus-4.5", "lab": "anthropic", "name": "Claude Opus 4.5", "family": "Claude Opus", "date": "2025-11-24", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-opus-4-5", "title": "Introducing Claude Opus 4.5", "quote": "Our newest model, Claude Opus 4.5, is available today.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "retrieved_at": "2026-09-07T05:00:00Z" } ], "scores": [ { "benchmark": "swe-bench-verified", "value": 80.9, "config": "no extended thinking", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "quote": "SWE-bench 80.9%4 77.2% 74.5% 76.2% 76.3%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 2.3.A overview; footnote 4 = 'Without extended thinking'. With 64k thinking, Opus 4.5 scores 80.60% (Table 2.4.A)." } }, { "benchmark": "terminal-bench-2", "value": 59.27, "config": "Terminal-Bench 2.0, Terminus-2 harness, 128k thinking budget", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "quote": "With a 128k thinking budget, Claude Opus 4.5 achieved a score of 59.27%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "gpqa-diamond", "value": 86.95, "config": "extended thinking, 64k thinking budget, default effort high", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "quote": "Claude Opus 4.5 achieved a score of 86.95% (averaged over 5 trials) on GPQA Diamond.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "mmmu", "value": 80.72, "config": "validation split, extended thinking 64k", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "quote": "Claude Opus 4.5 achieved a score of 80.72% on MMMU.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "aime", "value": 92.77, "config": "AIME 2025, no tools, extended thinking 64k", "note": "With access to Python tools, Anthropic reports 100%; contamination concerns are flagged for this figure in the system card.", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "quote": "Claude Opus 4.5 achieved a score of 92.77% without tools, and 100% with access to python tools.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "arc-agi-2", "value": 37.6, "config": "ARC Prize verified set", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "quote": "ARC-AGI-2 37.6% 13.6% — 31.1% 17.6%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 2.3.A overview row (Verified); 37.6% is Opus 4.5's value." } }, { "benchmark": "tau2-bench", "value": 91.63, "config": "no extended thinking; unweighted mean of retail, telecom and corrected-airline domains", "note": "Retail 88.9%, Telecom 98.2%, and a corrected Airline figure of 87.8% (the system card's originally-reported Airline score of 67.9% was later described by Anthropic as based on a flawed task version; 87.8% is Anthropic's own corrected figure, footnote 6).", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "quote": "88.9%4 86.2% 86.8% 85.3%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "lmarena-text", "value": 1473, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "34 20 47 claude-opus-4-5-20251101-high-32k Anthropic · Proprietary 1473 ±4 36,269", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "The marketing announcement page (anthropic.com/news/claude-opus-4-5) shows its benchmark comparisons only as images; all scores here are sourced from the text of the Claude Opus 4.5 System Card PDF (served at anthropic.com/claude-opus-4-5-system-card), Table 2.3.A and the corresponding narrative sections 2.16-2.20." }, { "id": "anthropic-claude-haiku-4.5", "lab": "anthropic", "name": "Claude Haiku 4.5", "family": "Claude Haiku", "date": "2025-12-01", "date_precision": "month", "status": "released", "tier": "small", "announcement": { "url": "https://www.anthropic.com/features/project-deal", "title": "Project Deal: our Claude-run marketplace experiment", "quote": "In December 2025, Claude interviewed people about which of their personal belongings they might want to sell and what sorts of things they might be willing to buy.", "retrieved_at": "2026-10-23T15:15:39Z", "verified": true, "verified_at": "2026-10-23T15:15:39Z" }, "scores": [], "notes": "The page refers to this as the \"smallest model\" used in an experiment conducted in December 2025." }, { "id": "deepseek-v3.2", "lab": "deepseek", "name": "DeepSeek-V3.2", "family": "DeepSeek-V", "date": "2025-12-01", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/raw/main/README.md", "title": "DeepSeek-V3.2", "quote": "We introduce DeepSeek-V3.2, a model that harmonizes high computational efficiency with superior reasoning and agent performance.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news251201/", "title": "DeepSeek-V3.2 Release", "retrieved_at": "2026-09-07T03:30:00Z" }, { "url": "https://arxiv.org/abs/2512.02556", "title": "DeepSeek-V3.2: Pushing the Frontier of Open Large Language Models (technical report)", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu-pro", "value": 85, "config": "EM, thinking-in-tool-use mode", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2512.02556v1", "title": "DeepSeek-V3.2 technical report, Table 2", "quote": "MMLU-Pro (EM) 88.2 87.5 90.1 84.6 82.0 85.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 82.4, "config": "pass@1, thinking-in-tool-use mode", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2512.02556v1", "title": "DeepSeek-V3.2 technical report, Table 2", "quote": "GPQA Diamond (Pass@1) 83.4 85.7 91.9 84.5 77.7 82.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hle", "value": 25.1, "config": "pass@1, thinking-in-tool-use mode", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2512.02556v1", "title": "DeepSeek-V3.2 technical report, Table 2", "quote": "HLE (Pass@1) 13.7 26.3 37.7 23.9 12.5 25.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "livecodebench", "value": 83.3, "config": "Pass@1-COT", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2512.02556v1", "title": "DeepSeek-V3.2 technical report, Table 2", "quote": "LiveCodeBench (Pass@1-COT) 64.0 84.5 90.7 82.6 83.0 83.3", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "aime", "value": 93.1, "config": "AIME 2025, pass@1", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2512.02556v1", "title": "DeepSeek-V3.2 technical report, Table 2", "quote": "AIME 2025 (Pass@1) 87.0 94.6 95.0 94.5 78.3 93.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "swe-bench-verified", "value": 73.1, "config": "Resolved", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2512.02556v1", "title": "DeepSeek-V3.2 technical report, Table 2", "quote": "SWE Verified (Resolved) 77.2 74.9 76.2 71.3 69.4 73.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "terminal-bench-2", "value": 46.4, "config": "Acc", "reported_by": "official", "source": { "url": "https://arxiv.org/html/2512.02556v1", "title": "DeepSeek-V3.2 technical report, Table 2", "quote": "Terminal Bench 2.0 (Acc) 42.8 35.2 54.2 35.7 30.0 46.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1425, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "108 94 124 deepseek-v3.2 DeepSeek · MIT 1425 ±4 46,493", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Official successor to V3.2-Exp; DeepSeek-V3.2-Speciale (API-only, maximum reasoning) was launched the same day as a separate reasoning-maximized variant, not recorded here as it targets research/eval use rather than general availability." }, { "id": "mistral-mistral-large-3", "lab": "mistral", "name": "Mistral Large 3", "family": "Mistral Large", "date": "2025-12-02", "date_precision": "day", "status": "released", "announcement": { "url": "https://mistral.ai/news/mistral-3/", "title": "Introducing Mistral 3", "quote": "Mistral 3 includes three state-of-the-art small, dense models (14B, 8B, and 3B) and Mistral Large 3 – our most capable model to date – a sparse mixture-of-experts trained with 41B active and 675B total parameters.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:30:07Z" }, "sources": [ { "url": "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512/raw/main/README.md", "title": "mistralai/Mistral-Large-3-675B-Instruct-2512 model card", "retrieved_at": "2026-09-07T04:35:00Z", "note": "README contains tool-use/function-calling code examples only, no benchmark table." }, { "url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "title": "Mistral Large 3 — model documentation", "retrieved_at": "2026-09-07T04:35:00Z", "note": "Client-rendered docs page; no GPQA/AIME/MMLU-Pro/HLE/ARC-AGI-2/SWE-bench/MMMU/LiveCodeBench/Terminal-Bench text or table found in the fetched HTML." } ], "scores": [ { "benchmark": "lmarena-text", "value": 1414, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "130 113 139 mistral-large-3 Mistral · Apache 2.0 1414 ±3 65,587", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "No official numeric benchmark scores could be sourced: the launch post's performance comparisons are rendered as images only ('Chart Base Models', 'Model Performance Comparison (instruct)', GPQA Diamond chart), the model's Hugging Face README has no benchmark table, the docs.mistral.ai model page has no extractable benchmark text, and no dedicated technical report for Mistral Large 3 was found as of the retrieval date (re-checked 2026-09-07). The launch post's only readable numeric capability figure (85% on AIME '25) belongs to the separate Ministral 14B reasoning variant, a non-flagship mini tier excluded by this project's rules." }, { "id": "openai-gpt-5.2", "lab": "openai", "name": "GPT-5.2", "family": "GPT", "date": "2025-12-11", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/introducing-gpt-5-2/", "title": "Introducing GPT-5.2", "quote": "In ChatGPT, GPT‑5.2 Instant, Thinking, and Pro will begin rolling out today, starting with paid plans.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "swe-bench-verified", "value": 80, "config": "GPT-5.2 Thinking", "note": "Appendix table, columns GPT-5.2 Thinking / GPT-5.2 Pro / GPT-5.1 Thinking.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-2/", "quote": "| SWE-bench Verified | 80.0% | - | 76.3% |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gpqa-diamond", "value": 92.4, "config": "GPT-5.2 Thinking, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-2/", "quote": "GPT‑5.2 Pro achieves 93.2%, followed closely by GPT‑5.2 Thinking at 92.4%.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "aime", "value": 100, "config": "AIME 2025, GPT-5.2 Thinking, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-2/", "quote": "| AIME 2025 (no tools) | 100.0% | 100.0% | 94.0% |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hle", "value": 34.5, "config": "GPT-5.2 Thinking, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-2/", "quote": "| HLE (no tools) | 34.5% | 36.6% | 25.7% |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 80.4, "config": "MMMU-Pro, with Python tool", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-2/", "quote": "| MMMU Pro (w/ Python) | 80.4% | - | 79.0% |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1436, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "88 79 105 gpt-5.2 OpenAI · Proprietary 1436 ±3 79,070", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "GPT-5.2 Thinking is the top general-purpose reasoning tier at launch (GPT-5.2 Pro scores marginally higher on some evals, e.g. GPQA Diamond 93.2%, and is recorded in config text where relevant)." }, { "id": "zhipu-glm-4.7", "lab": "zhipu", "name": "GLM-4.7", "family": "GLM", "date": "2025-12-22", "date_precision": "day", "status": "released", "announcement": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "GLM-4.7, your new coding partner, is coming with the following features:", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "mmlu-pro", "value": 84.3, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| MMLU-Pro | 84.3 | 83.2 | 84.6 | 85.0 | 90.1 | 88.2 | 87.5 | 87.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "gpqa-diamond", "value": 85.7, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| GPQA-Diamond | 85.7 | 81.0 | 84.5 | 82.4 | 91.9 | 83.4 | 85.7 | 88.1 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "hle", "value": 24.8, "config": "text-only", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| HLE | 24.8 | 17.2 | 23.9 | 25.1 | 37.5 | 13.7 | 26.3 | 25.7 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "aime", "value": 95.7, "config": "AIME 2025", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| AIME 2025 | 95.7 | 93.9 | 94.5 | 93.1 | 95.0 | 87.0 | 94.6 | 94.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "livecodebench", "value": 84.9, "config": "v6", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| LiveCodeBench-v6 | 84.9 | 82.8 | 83.1 | 83.3 | 90.7 | 64.0 | 87.0 | 87.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "swe-bench-verified", "value": 73.8, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| SWE-bench Verified | 73.8 | 68.0 | 71.3 | 73.1 | 76.2 | 77.2 | 74.9 | 76.3 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "terminal-bench-2", "value": 41, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| Terminal Bench 2.0 | 41.0 | 24.5 | 35.7 | 46.4 | 54.2 | 42.8 | 35.2 | 47.6 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "tau2-bench", "value": 87.4, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-4.7", "title": "GLM-4.7: Advancing the Coding Capability", "quote": "| τ²-Bench | 87.4 | 75.2 | 74.3 | 85.3 | 90.7 | 87.2 | 82.4 | 82.7 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "17-benchmark comparison table published in the GLM-4.7 launch post; columns are GLM-4.7, GLM-4.6, Kimi K2 Thinking, DeepSeek-V3.2, Gemini 3.0 Pro, Claude Sonnet 4.5, GPT-5 High, GPT-5.1 High." } }, { "benchmark": "lmarena-text", "value": 1442, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "81 64 100 glm-4.7 Z.ai · MIT 1442 ±6 11,889", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "xai-grok-5", "lab": "xai", "name": "Grok 5", "family": "Grok", "date": "2026-01-01", "date_precision": "quarter", "status": "announced", "expected_window": { "start": "2026-01-01", "end": "2026-03-31", "source": { "url": "https://www.theinformation.com/briefings/elon-musks-xai-delays-grok-5-release-next-year", "title": "Elon Musk's xAI Delays Grok 5 Release to Next Year", "quote": "XAI plans to release its next-generation Grok 5 artificial intelligence model in the first three months of next year, CEO Elon Musk said on Friday.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:30:58Z", "via": "r.jina.ai", "note": "Article body is paywalled; quote is taken from the page's own meta description / teaser text, which is present in the fetched HTML regardless of subscription status." } }, "announcement": { "url": "https://www.theinformation.com/briefings/elon-musks-xai-delays-grok-5-release-next-year", "title": "Elon Musk's xAI Delays Grok 5 Release to Next Year", "quote": "XAI plans to release its next-generation Grok 5 artificial intelligence model in the first three months of next year, CEO Elon Musk said on Friday.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": false, "verified_at": "2026-09-07T09:30:58Z", "via": "r.jina.ai", "note": "Article body is paywalled; quote is taken from the page's own meta description / teaser text, which is present in the fetched HTML regardless of subscription status." }, "scores": [], "notes": "Reported by The Information (Theo Wayt, Nov 14, 2025): in an interview with investor Ron Baron, Musk said Grok 5 would ship 'in Q1' 2026 (6 trillion parameters, native video understanding), later than his earlier end-of-2025 target. That Q1 2026 window has since passed without a release; by mid-2026 xAI shipped intermediate models (Grok 4.5, Grok 4.6) instead, and public reporting as of Aug 2026 places Grok 5 training on the newer 'Colossus 2' cluster with no firm date — xAI has not published a revised official date, so expected_window above reflects only the original Nov 2025 statement and is now stale." }, { "id": "moonshot-kimi-k2.5", "lab": "moonshot", "name": "Kimi K2.5", "family": "Kimi K", "date": "2026-01-27", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "Kimi K2.5 is an open-source, native multimodal agentic model built through continual pretraining on approximately 15 trillion mixed visual and text tokens atop Kimi-K2-Base.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" }, "scores": [ { "benchmark": "hle", "value": 30.1, "config": "HLE-Full, no tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "HLE-Full 30.1 34.5 30.8 37.5 25.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Table columns: Kimi K2.5 (Thinking), GPT-5.2 (xhigh), Claude 4.5 Opus (Extended Thinking), Gemini 3 Pro (High Thinking), DeepSeek V3.2 (Thinking)." } }, { "benchmark": "aime", "value": 96.1, "config": "AIME 2025", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "AIME 2025 96.1 100 92.8 95.0 93.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "gpqa-diamond", "value": 87.6, "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "GPQA-Diamond 87.6 92.4 87.0 91.9 82.4", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "mmlu-pro", "value": 87.1, "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "MMLU-Pro 87.1 86.7* 89.3* 90.1 85.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "livecodebench", "value": 85, "config": "v6", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "LiveCodeBench (v6) 85.0 - 82.2* 87.4* 83.3", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "swe-bench-verified", "value": 76.8, "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "SWE-Bench Verified 76.8 80.0 80.9 76.2", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "terminal-bench-2", "value": 50.8, "config": "Terminus-2, non-thinking mode", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "title": "moonshotai/Kimi-K2.5 model card", "quote": "Terminal Bench 2.0 50.8 54.0 59.3 54.2", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Lab notes Terminal-Bench 2.0 was evaluated under non-thinking mode because their context-management strategy for thinking mode is incompatible with Terminus-2." } }, { "benchmark": "lmarena-text", "value": 1451, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "67 52 79 kimi-k2.5-thinking Moonshot · Modified MIT 1451 ±3 70,586", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "anthropic-claude-opus-4.6", "lab": "anthropic", "name": "Claude Opus 4.6", "family": "Claude Opus", "date": "2026-02-05", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-opus-4-6", "title": "Claude Opus 4.6", "quote": "Claude Opus 4.6 is available today on claude.ai, our API, and all major cloud platforms.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www.anthropic.com/claude-opus-4-6-system-card", "title": "Claude Opus 4.6 System Card", "retrieved_at": "2026-09-07T05:00:00Z" } ], "scores": [ { "benchmark": "swe-bench-verified", "value": 80.8, "config": "adaptive thinking, max effort, averaged over 25 trials", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-6-system-card", "title": "Claude Opus 4.6 System Card", "quote": "SWE-bench 80.8% 80.9% 77.2% 76.2% 80.0%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 2.3.A, columns Opus 4.6 / Opus 4.5 / Sonnet 4.5 / Gemini 3 Pro / GPT-5.2; 80.8% is Opus 4.6's value." } }, { "benchmark": "terminal-bench-2", "value": 65.4, "config": "adaptive thinking, max effort", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-6-system-card", "title": "Claude Opus 4.6 System Card", "quote": "Terminal-Bench 65.4% 59.8% 51.0% 56.2% 64.7%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 2.3.A Terminal-Bench 2.0 row; 65.4% is Opus 4.6's value." } }, { "benchmark": "gpqa-diamond", "value": 91.3, "config": "adaptive thinking, max effort", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-6-system-card", "title": "Claude Opus 4.6 System Card", "quote": "GPQA Diamond 91.3% 87.0% 83.40% 91.9% 93.2%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 2.3.A; 91.3% is Opus 4.6's value." } }, { "benchmark": "mmmu", "value": 73.9, "config": "MMMU-Pro, no tools", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-6-system-card", "title": "Claude Opus 4.6 System Card", "quote": "MMMU-Pro (no tools) 73.9% 70.6% 63.4% 81% 79.5%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "arc-agi-2", "value": 68.8, "config": "ARC Prize verified set, deep thinking", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-6-system-card", "title": "Claude Opus 4.6 System Card", "quote": "ARC-AGI-2 68.8% 37.6% 13.6% 45.1% 54.2%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "tau2-bench", "value": 95.6, "config": "unweighted mean of retail and telecom domains (airline not reported for this release)", "note": "Retail 91.9%, Telecom 99.3%; average of the two domains reported.", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-6-system-card", "title": "Claude Opus 4.6 System Card", "quote": "91.9% 88.9% 86.2% 85.3% 82.0%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "lmarena-text", "value": 1498, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "6 3 11 claude-opus-4-6 Anthropic · Proprietary 1498 ±3 76,015", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "First Opus model with a 1M-token context window (beta). Launch date corroborated by CNBC (https://www.cnbc.com/2026/02/05/anthropic-claude-opus-4-6-vibe-working.html); the system card itself only states 'February 2026'." }, { "id": "zhipu-glm-5", "lab": "zhipu", "name": "GLM-5", "family": "GLM", "date": "2026-02-11", "date_precision": "day", "status": "released", "announcement": { "url": "https://z.ai/blog/glm-5", "title": "GLM-5: From Vibe Coding to Agentic Engineering", "quote": "We are launching GLM-5, targeting complex systems engineering and long-horizon agentic tasks.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" }, "sources": [ { "url": "https://www.reuters.com/technology/chinas-ai-startup-zhipu-releases-new-flagship-model-glm-5-2026-02-11/", "title": "Chinese AI startup Zhipu releases new flagship model GLM-5 (Reuters, 11 Feb 2026)", "retrieved_at": "2026-09-07T03:26:49Z", "note": "Used to pin exact launch day; article URL itself is dated 2026-02-11." } ], "scores": [ { "benchmark": "hle", "value": 30.5, "config": "text-only", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5", "title": "GLM-5: From Vibe Coding to Agentic Engineering", "quote": "| Humanity's Last Exam | 30.5 | 24.8 | 25.1 | 31.5 | 28.4 | 37.2 | 35.4 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "gpqa-diamond", "value": 86, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5", "title": "GLM-5: From Vibe Coding to Agentic Engineering", "quote": "| GPQA-Diamond | 86.0 | 85.7 | 82.4 | 87.6 | 87.0 | 91.9 | 92.4 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "aime", "value": 92.7, "config": "AIME 2026 I", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5", "title": "GLM-5: From Vibe Coding to Agentic Engineering", "quote": "| AIME 2026 I | 92.7 | 92.9 | 92.7 | 92.5 | 93.3 | 90.6 | - |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "swe-bench-verified", "value": 77.8, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5", "title": "GLM-5: From Vibe Coding to Agentic Engineering", "quote": "| SWE-bench Verified | 77.8 | 73.8 | 73.1 | 76.8 | 80.9 | 76.2 | 80.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "terminal-bench-2", "value": 56.2, "config": "Terminus-2 harness", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5", "title": "GLM-5: From Vibe Coding to Agentic Engineering", "quote": "| Terminal-Bench 2.0 Terminus-2 | 56.2 / 60.7† | 41.0 | 39.3 | 50.8 | 59.3 | 54.2 | 54.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "Standard Terminus-2 harness score; a lab-verified variant of the benchmark (fixing ambiguous instructions) scored 60.7." } }, { "benchmark": "tau2-bench", "value": 89.7, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5", "title": "GLM-5: From Vibe Coding to Agentic Engineering", "quote": "| τ²-Bench | 89.7 | 87.4 | 85.3 | 80.2 | 91.6 | 90.7 | 85.5 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "lmarena-text", "value": 1458, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "56 44 68 glm-5 Z.ai · MIT 1458 ±4 27,647", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "google-gemini-3.1-pro", "lab": "google", "name": "Gemini 3.1 Pro", "family": "Gemini Pro", "date": "2026-02-19", "date_precision": "day", "status": "released", "announcement": { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/", "title": "Gemini 3.1 Pro: A smarter model for your most complex tasks", "quote": "Today, we’re releasing the upgraded core intelligence that makes those breakthroughs possible: Gemini 3.1 Pro.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "sources": [ { "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-3-1-Pro-Model-Card.pdf", "title": "Gemini 3.1 Pro Model Card", "retrieved_at": "2026-09-07T07:45:00Z", "note": "Capability-benchmark table is an embedded image with no extractable text." }, { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "retrieved_at": "2026-09-07T04:09:50Z", "note": "Live model page renders the benchmark comparison table (Gemini 3.1 Pro vs Gemini 3 Pro vs competitors) as plain HTML markup, unlike the PDF model card and the launch blog post which only show it as an image." } ], "scores": [ { "benchmark": "arc-agi-2", "value": 77.1, "config": "ARC Prize verified", "reported_by": "official", "source": { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/", "title": "Gemini 3.1 Pro: A smarter model for your most complex tasks", "quote": "On ARC-AGI-2, a benchmark that evaluates a model’s ability to solve entirely new logic patterns, 3.1 Pro achieved a verified score of 77.1%.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "hle", "value": 44.4, "config": "Thinking (High), no tools", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "gpqa-diamond", "value": 94.3, "config": "Thinking (High), no tools", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "terminal-bench-2", "value": 68.5, "config": "Thinking (High), Terminus-2 harness", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "swe-bench-verified", "value": 80.6, "config": "Thinking (High), single attempt", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "tau2-bench", "value": 95.05, "config": "Thinking (High); unweighted mean of retail and telecom domains (airline not reported for this release)", "note": "Retail 90.8%, Telecom 99.3% (Gemini 3.1 Pro Thinking (High) column); average of the two domains reported, per this project's methodology for domain-split τ2-bench scores.", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "τ2-bench Agentic and tool use", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "swe-bench-pro", "value": 54.2, "config": "Thinking (High), single attempt, public set", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "lmarena-text", "value": 1487, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "15 7 24 gemini-3.1-pro-preview Google · Proprietary 1487 ±3 102,999", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Launched in preview (Gemini API in AI Studio, Gemini CLI, Google Antigravity, Android Studio; Vertex AI and Gemini Enterprise for business; Gemini app and NotebookLM for consumers) — not yet GA at launch, per the post's own 'before we make it generally available soon'. Counted as released per project convention since it was immediately publicly usable, mirroring the treatment of Gemini 2.0/2.5 Pro Experimental. The launch post itself gives only the ARC-AGI-2 number and the PDF model card's benchmark table is an image, but the live deepmind.google/models/gemini/pro comparison page (re-fetched 2026-09-07) renders the same benchmark table as plain HTML, from which HLE, GPQA Diamond, Terminal-Bench 2.0, SWE-bench Verified, τ2-bench (Retail/Telecom) and MMMU-Pro were recovered — resolving the gap flagged in earlier research on this release. LiveCodeBench is reported on that page only as 'LiveCodeBench Pro' Elo rating (2887), a different scale incompatible with this project's 0-100 pass@1 metric, so it is not recorded." }, { "id": "openai-gpt-5.4", "lab": "openai", "name": "GPT-5.4", "family": "GPT", "date": "2026-03-05", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/introducing-gpt-5-4/", "title": "Introducing GPT-5.4", "quote": "GPT‑5.4 is rolling out gradually today across ChatGPT and Codex.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "gpqa-diamond", "value": 92.8, "config": "no tools", "note": "Table columns: GPT-5.4 / GPT-5.4 Pro / GPT-5.3-Codex / GPT-5.2 / GPT-5.2 Pro.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-4/", "quote": "GPQA Diamond 92.8%94.4%92.6%92.4%93.2%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hle", "value": 39.8, "config": "no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-4/", "quote": "Humanity's Last Exam (no tools)39.8%42.7%—34.5%36.6%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 81.2, "config": "MMMU-Pro, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-4/", "quote": "On MMMU-Pro, a test of a model's visual understanding and reasoning, GPT‑5.4 achieves an 81.2% success rate without tool use, an improvement over GPT‑5.2's 79.5%.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "terminal-bench-2", "value": 75.1, "config": "Terminal-Bench 2.0", "note": "Table columns: GPT-5.4 / GPT-5.4 Pro / GPT-5.3-Codex / GPT-5.2 / GPT-5.2 Pro.", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-4/", "quote": "Terminal-Bench 2.0 75.1%—77.3%62.2%—", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1466, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "46 29 57 gpt-5.4 OpenAI · Proprietary 1466 ±4 63,588", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "First mainline GPT-5.x reasoning model to incorporate GPT-5.3-Codex's coding capabilities; classic AIME/SWE-bench Verified were not reported in this launch post's appendix (GDPval, SWE-Bench Pro and OSWorld-Verified were emphasized instead)." }, { "id": "zhipu-glm-5.1", "lab": "zhipu", "name": "GLM-5.1", "family": "GLM", "date": "2026-04-07", "date_precision": "day", "status": "released", "announcement": { "url": "https://z.ai/blog/glm-5.1", "title": "GLM-5.1: Towards Long-Horizon Tasks", "quote": "GLM-5.1 is our next-generation flagship model for agentic engineering, with significantly stronger coding capabilities than its predecessor.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "hle", "value": 31, "config": "text-only", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.1", "title": "GLM-5.1: Towards Long-Horizon Tasks", "quote": "| HLE | 31.0 | 30.5 | 28.8 | 28.0 | 25.1 | 31.5 | 36.7 | 45.0 | 39.8 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "Row order: GLM-5.1, GLM-5, Qwen3.6-Plus, MiniMax M2.7, DeepSeek-V3.2, Kimi K2.5, Claude Opus 4.6, Gemini 3.1 Pro, GPT-5.4." } }, { "benchmark": "aime", "value": 95.3, "config": "AIME 2026", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.1", "title": "GLM-5.1: Towards Long-Horizon Tasks", "quote": "| AIME 2026 | 95.3 | 95.4 | 95.1 | 89.8 | 95.1 | 94.5 | 95.6 | 98.2 | 98.7 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "gpqa-diamond", "value": 86.2, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.1", "title": "GLM-5.1: Towards Long-Horizon Tasks", "quote": "| GPQA-Diamond | 86.2 | 86.0 | 90.4 | 87.0 | 82.4 | 87.6 | 91.3 | 94.3 | 92.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "terminal-bench-2", "value": 63.5, "config": "Terminus-2 harness", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.1", "title": "GLM-5.1: Towards Long-Horizon Tasks", "quote": "| Terminal-Bench 2.0 Terminus-2 | 63.5 | 56.2 | 61.6 | - | 39.3 | 50.8 | 65.4 | 68.5 | - |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "swe-bench-pro", "value": 58.4, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.1", "title": "GLM-5.1: Towards Long-Horizon Tasks", "quote": "| SWE-Bench Pro | 58.4 | 55.1 | 56.6 | 56.2 | - | 53.8 | 57.3 | 54.2 | 57.7 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "lmarena-text", "value": 1466, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "44 28 56 glm-5.1 Z.ai · MIT 1466 ±4 45,650", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "meta-muse-spark", "lab": "meta", "name": "Muse Spark", "family": "Muse", "date": "2026-04-08", "date_precision": "day", "status": "released", "announcement": { "url": "https://ai.meta.com/blog/introducing-muse-spark-msl/", "title": "Introducing Muse Spark: Scaling Towards Personal Superintelligence", "quote": "Today, we're excited to introduce Muse Spark, the first in the Muse family of models developed by Meta Superintelligence Labs. Muse Spark is a natively multimodal reasoning model with support for tool-use, visual chain of thought, and multi-agent orchestration.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" }, "sources": [ { "url": "https://about.fb.com/news/2026/04/introducing-muse-spark-meta-superintelligence-labs/", "title": "Introducing Muse Spark: Meta's Most Powerful Model Yet", "retrieved_at": "2026-09-07T03:30:00Z" }, { "url": "https://ai.meta.com/static-resource/muse-spark-eval-methodology", "title": "Muse Spark Eval Methodology", "retrieved_at": "2026-09-07T04:25:00Z", "note": "Describes how each benchmark (HLE, GPQA Diamond, ARC-AGI-2, SWE-Bench Verified/Pro, Terminal-Bench 2.0, LiveCodeBench Pro, tau-Bench Telecom, GDPval-AA) was run, but the document itself reports no actual scores/results table." }, { "url": "https://ai.meta.com/static-resource/muse-spark-safety-and-preparedness-report/", "title": "Muse Spark Safety & Preparedness Report", "retrieved_at": "2026-09-07T04:25:00Z", "note": "160-page frontier-risk framework report (Chemical/Bio, Cyber, Loss of Control domains). Only non-standard variants appear (e.g. 'SWE-Bench Verified (hard subset)', 'Terminal-Bench Pro' 68.7%), none matching this project's tracked benchmark configs; no general GPQA/ARC-AGI-2/MMLU-Pro/AIME/SWE-bench Verified leaderboard table." }, { "url": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report/", "title": "Muse Spark 1.1 Evaluation Report", "retrieved_at": "2026-09-07T04:25:00Z", "note": "July 9, 2026 update to the same Muse Spark checkpoint family; likewise structured as a safety/preparedness report with no general-capability leaderboard table. Reports 'resolved 24/42 unique tasks on SWE-Bench Verified Hard' (a dangerous-capability threshold metric, not this project's tracked pass@1 SWE-bench Verified config)." } ], "scores": [ { "benchmark": "hle", "value": 58, "config": "Contemplating mode (multi-agent test-time scaling)", "note": "Meta's launch post gives this single headline number for HLE via its extended \"Contemplating mode\"; no plain/no-tools HLE figure or other index-basket scores (MMLU-Pro, GPQA, SWE-bench, etc.) were published in the launch post or found in an official model card at time of writing. Re-checked 2026-09-07 against the Eval Methodology doc and the Safety & Preparedness Reports for Muse Spark 1.0 and 1.1 (see sources[]) — neither contains a general-capability comparison table with index-basket numbers.", "reported_by": "official", "source": { "url": "https://ai.meta.com/blog/introducing-muse-spark-msl/", "title": "Introducing Muse Spark: Scaling Towards Personal Superintelligence", "quote": "Contemplating mode provides significant capability improvements in challenging tasks, achieving 58% in Humanity's Last Exam and 38% in FrontierScience Research.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:29:10Z", "via": "r.jina.ai" } }, { "benchmark": "lmarena-text", "value": 1488, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "13 5 25 muse-spark Meta · Proprietary 1488 ±6 13,572", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "First model from Meta Superintelligence Labs (led by Alexandr Wang), closed-weight, available via meta.ai / Meta AI app with a private API preview. Point updates (Muse Spark 1.1, ~May 2026; reportedly 1.3 later) rolled the same model out further (voice, glasses, shopping) rather than establishing a new flagship generation, so they are treated as updates to this release rather than separate entries." }, { "id": "moonshot-kimi-k2.6", "lab": "moonshot", "name": "Kimi K2.6", "family": "Kimi K", "date": "2026-04-21", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.kimi.com/blog/kimi-k2-6.html", "title": "Kimi K2.6 Tech Blog: Advancing Open-Source Coding", "quote": "We are open sourcing our latest model, Kimi K2.6, featuring state-of-the-art coding, long-horizon execution, and agent swarm capabilities. Kimi K2.6 is now available via Kimi.ai, the Kimi App, the API, and Kimi Code.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Blog page carries no visible publish date; Apr 21, 2026 GA date corroborated by the Hugging Face repo creation date (Apr 14, 2026, matching the earlier 'Code Preview') and third-party coverage of the GA announcement." }, "scores": [ { "benchmark": "terminal-bench-2", "value": 66.7, "config": "Terminus-2, preserve-thinking mode", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.6", "title": "moonshotai/Kimi-K2.6 model card", "quote": "Terminal-Bench 2.0 (Terminus-2) 66.7 65.4* 65.4 68.5 50.8", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Table columns: Kimi K2.6, GPT-5.4 (xhigh), Claude Opus 4.6 (max effort), Gemini 3.1 Pro (thinking high), Kimi K2.5." } }, { "benchmark": "swe-bench-verified", "value": 80.2, "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.6", "title": "moonshotai/Kimi-K2.6 model card", "quote": "SWE-Bench Verified 80.2 - 80.8 80.6 76.8", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "livecodebench", "value": 89.6, "config": "v6", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.6", "title": "moonshotai/Kimi-K2.6 model card", "quote": "LiveCodeBench (v6) 89.6 - 88.8 91.7 85.0", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "hle", "value": 34.7, "config": "HLE-Full, no tools", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.6", "title": "moonshotai/Kimi-K2.6 model card", "quote": "HLE-Full 34.7 39.8 40.0 44.4 30.1", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "aime", "value": 96.4, "config": "AIME 2026", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.6", "title": "moonshotai/Kimi-K2.6 model card", "quote": "AIME 2026 96.4 99.2 96.7 98.3 95.8", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "gpqa-diamond", "value": 90.5, "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K2.6", "title": "moonshotai/Kimi-K2.6 model card", "quote": "GPQA-Diamond 90.5 92.8 91.3 94.3 87.6", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z" } }, { "benchmark": "lmarena-text", "value": 1461, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "50 39 66 kimi-k2.6 Moonshot · Modified MIT 1461 ±5 37,568", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "openai-gpt-5.5", "lab": "openai", "name": "GPT-5.5", "family": "GPT", "date": "2026-04-23", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/introducing-gpt-5-5/", "title": "Introducing GPT-5.5", "quote": "Today, GPT‑5.5 is rolling out to Plus, Pro, Business, and Enterprise users in ChatGPT and Codex, and GPT‑5.5 Pro is rolling out to Pro, Business, and Enterprise users in ChatGPT.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "terminal-bench-2", "value": 82.7, "config": "Terminal-Bench 2.0", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-5/", "quote": "On Terminal-Bench 2.0, which tests complex command-line workflows requiring planning, iteration, and tool coordination, it achieves a state-of-the-art accuracy of 82.7%.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "gpqa-diamond", "value": 93.6, "config": "no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-5/", "quote": "GPQA Diamond 93.6%92.8%—94.4%94.2%94.3%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hle", "value": 41.4, "config": "no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-5/", "quote": "Humanity's Last Exam (no tools)41.4%39.8%43.1%42.7%46.9%44.4%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 81.2, "config": "MMMU-Pro, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-5-5/", "quote": "MMMU Pro (no tools)81.2%81.2%——80.5%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1477, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "24 17 39 gpt-5.5 OpenAI · Proprietary 1477 ±4 64,977", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Launch post's evaluation table lists competing models including 'Opus 4.6' as a comparison column, corroborating Anthropic's Opus 4.x line was still current in April 2026." }, { "id": "deepseek-v4-pro", "lab": "deepseek", "name": "DeepSeek-V4-Pro", "family": "DeepSeek-V", "date": "2026-04-24", "date_precision": "day", "status": "released", "announcement": { "url": "https://api-docs.deepseek.com/news/news260424/", "title": "DeepSeek V4 Preview Release", "quote": "DeepSeek-V4 Preview is officially live & open-sourced! Welcome to the era of cost-effective 1M context length.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" }, "sources": [ { "url": "https://api-docs.deepseek.com/news/news260813/", "title": "DeepSeek-V4-Pro GA Release", "retrieved_at": "2026-09-07T03:30:00Z" }, { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "title": "DeepSeek-V4-Pro model card", "retrieved_at": "2026-09-07T03:30:00Z" } ], "scores": [ { "benchmark": "mmlu-pro", "value": 87.5, "config": "EM, Think Max reasoning mode", "note": "Scores are from the current (post-GA, V4-Pro-0813) model card, which reports the DeepSeek-V4-Pro-Max configuration; the model was first made available at preview on 2026-04-24 and reached GA as V4-Pro-0813 on 2026-08-13.", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "title": "DeepSeek-V4-Pro model card", "quote": "| MMLU-Pro (EM) | 89.1 | 87.5 | **91.0** | 87.1 | 86.0 | 87.5 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "gpqa-diamond", "value": 90.1, "config": "pass@1, Think Max reasoning mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "title": "DeepSeek-V4-Pro model card", "quote": "| GPQA Diamond (Pass@1) | 91.3 | 93.0 | **94.3** | 90.5 | 86.2 | 90.1 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "hle", "value": 37.7, "config": "pass@1, Think Max reasoning mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "title": "DeepSeek-V4-Pro model card", "quote": "| HLE (Pass@1) | 40.0 | 39.8 | **44.4** | 36.4 | 34.7 | 37.7 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "livecodebench", "value": 93.5, "config": "pass@1, Think Max reasoning mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "title": "DeepSeek-V4-Pro model card", "quote": "| LiveCodeBench (Pass@1) | 88.8 | - | 91.7 | 89.6 | - | **93.5** |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "terminal-bench-2", "value": 67.9, "config": "Acc, Think Max reasoning mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "title": "DeepSeek-V4-Pro model card", "quote": "| Terminal Bench 2.0 (Acc) | 65.4 | **75.1** | 68.5 | 66.7 | 63.5 | 67.9 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "swe-bench-verified", "value": 80.6, "config": "Resolved, Think Max reasoning mode", "reported_by": "official", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "title": "DeepSeek-V4-Pro model card", "quote": "| SWE Verified (Resolved) | **80.8** | - | 80.6 | 80.2 | - | 80.6 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:20Z" } }, { "benchmark": "lmarena-text", "value": 1458, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "55 45 68 deepseek-v4-pro DeepSeek · MIT 1458 ±4 54,225", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "1.6T-parameter (49B active) MoE model; DeepSeek-V4-Flash (284B/13B active) launched the same day as a smaller tier and is excluded here. Preview weights went live and open-sourced 2026-04-24; reasoning-effort modes (low/high/max) and the V4-Pro-0813 checkpoint shipped at GA on 2026-08-13. deepseek-chat/deepseek-reasoner API aliases were retired in favor of deepseek-v4-* on 2026-07-24." }, { "id": "google-gemini-3.5-pro", "lab": "google", "name": "Gemini 3.5 Pro", "family": "Gemini Pro", "date": "2026-05-19", "date_precision": "day", "status": "announced", "announcement": { "url": "https://techcrunch.com/2026/07/21/google-releases-three-new-gemini-models-but-no-3-5-pro/", "title": "Google releases three new Gemini models — but no 3.5 Pro", "quote": "Google DeepMind product lead Logan Kilpatrick said Tuesday that the company is currently testing Gemini 3.5 Pro with partners and hopes to “land soon.”", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "scores": [], "notes": "Teased at Google I/O (May 19, 2026) alongside the Gemini 3.5 Flash launch, with Google saying Pro was 'already being used internally' and would roll out 'next month'. As of this writing (Sep 7, 2026) it has missed at least three targets (June, mid-July, early August 2026); Bloomberg reported (Jul 16, 2026) that its coding performance was falling short of Google's internal goals. No model ID, pricing or firm launch date has been published, so no expected_window is recorded. Still unreleased and untested, hence no scores." }, { "id": "anthropic-claude-opus-4.8", "lab": "anthropic", "name": "Claude Opus 4.8", "family": "Claude Opus", "date": "2026-05-28", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-opus-4-8", "title": "Introducing Claude Opus 4.8", "quote": "Claude Opus 4.8 is available everywhere today.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www.anthropic.com/claude-opus-4-8-system-card", "title": "Claude Opus 4.8 System Card", "retrieved_at": "2026-09-07T05:00:00Z" } ], "scores": [ { "benchmark": "swe-bench-verified", "value": 88.6, "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-8-system-card", "title": "Claude Opus 4.8 System Card", "quote": "SWE-bench Verified 88.6 87.6 - 80.6", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 8.1 (Evaluation summary), columns Opus 4.8 / Opus 4.7 / GPT-5.5 / Gemini 3.1 Pro; 88.6 is Opus 4.8's value." } }, { "benchmark": "gpqa-diamond", "value": 93.6, "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-8-system-card", "title": "Claude Opus 4.8 System Card", "quote": "GPQA Diamond 93.6 94.2 - 94.3", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z" } }, { "benchmark": "hle", "value": 49.8, "config": "no tools", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-opus-4-8-system-card", "title": "Claude Opus 4.8 System Card", "quote": "No tools 49.8 46.9 41.4 44.4", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 8.1 Humanity's Last Exam row (no-tools variant)." } }, { "benchmark": "lmarena-text", "value": 1473, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "33 20 48 claude-opus-4-8 Anthropic · Proprietary 1473 ±4 49,404", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "System card title page date (May 28, 2026) is exact, dated the same day as the launch post. Anthropic's own Table 8.1 note: 'Changes to the Opus 4.7 BrowseComp/OSWorld scores' imply Opus 4.7 existed as an immediate predecessor (reported ~April 2026 per press, e.g. technology.org); Opus 4.7 is not recorded as its own release in this file given its very short 41-day lifespan and the research time budget for this pass. This system card switches to Terminal-Bench 2.1 (not 2.0), so no terminal-bench-2 score is recorded here to avoid mixing incompatible benchmark versions." }, { "id": "anthropic-claude-fable-5", "lab": "anthropic", "name": "Claude Fable 5", "family": "Claude Fable", "date": "2026-06-09", "date_precision": "day", "status": "released", "announcement": { "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "title": "Claude Fable 5 and Claude Mythos 5", "quote": "Claude Fable 5 is available everywhere today. Claude Mythos 5 is restricted to Glasswing partners (with cyber safeguards lifted) and soon to select biology researchers (with biology and chemistry safeguards lifted) only, until our broader trusted access program is available.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www.anthropic.com/claude-fable-5-system-card", "title": "System Card: Claude Fable 5 & Claude Mythos 5", "retrieved_at": "2026-09-07T05:00:00Z" }, { "url": "https://www.anthropic.com/news/redeploying-fable-5", "title": "Redeploying Claude Fable 5", "retrieved_at": "2026-09-07T05:00:00Z", "via": "r.jina.ai" } ], "scores": [ { "benchmark": "swe-bench-verified", "value": 95, "config": "adaptive thinking, max effort", "reported_by": "official", "source": { "url": "https://www.anthropic.com/claude-fable-5-system-card", "title": "System Card: Claude Fable 5 & Claude Mythos 5", "quote": "SWE-bench Verified 95.5 95 93.9 88.6 - 80.6", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 8.1.A, columns Mythos 5 / Fable 5 / Mythos Preview / Opus 4.8 / GPT-5.2 / Gemini 3.1 Pro; 95 is Fable 5's value (Mythos 5 itself scores 95.5)." } }, { "benchmark": "lmarena-text", "value": 1507, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "1 1 6 claude-fable-5 Anthropic · Proprietary 1507 ±5 27,189", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Anthropic describes Fable 5 and Mythos 5 as 'the same underlying model' (redeploying-fable-5 post): Fable 5 ships with production safeguards for cybersecurity/biology-chemistry/distillation topics that fall back to Claude Opus 4.8, while Mythos 5 has those safeguards lifted and is restricted to vetted Project Glasswing partners (not a public release, so not recorded as its own entry here). On June 12, 2026 the US government briefly applied export controls to both models, suspending access; access was restored June 30-July 1, 2026 (see the 'Redeploying Claude Fable 5' source). This generation's system card treats GPQA Diamond as saturated and stopped reporting it for Fable 5 specifically (Mythos 5 alone is given at 94.1%, 'averaged over 5 trials'); MMLU/MATH/HumanEval/MMMU are no longer reported for either model, and Terminal-Bench has moved to version 2.1 (not this basket's 2.0), so none of those appear as scores here. Family is recorded as 'Claude Fable' per this project's lineage-fork convention, even though the Opus line (Claude Opus 4.8, and subsequently a further 'Claude Opus 5', per the Fable 5.1 system card's comparison table) continued in parallel rather than being replaced." }, { "id": "zhipu-glm-5.2", "lab": "zhipu", "name": "GLM-5.2", "family": "GLM", "date": "2026-06-16", "date_precision": "day", "status": "released", "announcement": { "url": "https://z.ai/blog/glm-5.2", "title": "GLM-5.2: Built for Long-Horizon Tasks", "quote": "GLM-5.2, our latest flagship model for long-horizon tasks. It marks a substantial leap in long-horizon task capability over its predecessor GLM-5.1 and, for the first time, delivers that capability on a solid 1M-token context.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "hle", "value": 40.5, "config": "text-only", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.2", "title": "GLM-5.2: Built for Long-Horizon Tasks", "quote": "| HLE | 40.5 | 31.0 | 41.4 | 37.0 | 37.7 | 49.8* | 41.4* | 45.0 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "Row order: GLM-5.2, GLM-5.1, Qwen3.7-Max, MiniMax M3, DeepSeek-V4-Pro, Claude Opus 4.8, GPT-5.5, Gemini 3.1 Pro." } }, { "benchmark": "aime", "value": 99.2, "config": "AIME 2026", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.2", "title": "GLM-5.2: Built for Long-Horizon Tasks", "quote": "| AIME 2026 | 99.2 | 95.3 | 97.0 | - | 94.6 | 95.7 | 98.3 | 98.2 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "gpqa-diamond", "value": 91.2, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.2", "title": "GLM-5.2: Built for Long-Horizon Tasks", "quote": "| GPQA-Diamond | 91.2 | 86.2 | 90.0 | 93.0 | 90.1 | 93.6 | 93.6 | 94.3 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "swe-bench-pro", "value": 62.1, "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.2", "title": "GLM-5.2: Built for Long-Horizon Tasks", "quote": "| SWE-bench Pro | 62.1 | 58.4 | 60.6 | 59.0 | 55.4 | 69.2 | 58.6 | 54.2 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" } }, { "benchmark": "lmarena-text", "value": 1472, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "37 20 51 glm-5.2-max Z.ai · MIT 1472 ±5 33,966", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "GLM-5.2 reports 'Terminal Bench 2.1' rather than the 2.0 version tracked in this dataset's basket, so that score is not recorded here." }, { "id": "anthropic-claude-sonnet-5", "lab": "anthropic", "name": "Claude Sonnet 5", "family": "Claude Sonnet", "date": "2026-06-30", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://www.anthropic.com", "title": "Claude Fable 5.1", "quote": "Introducing Sonnet 5 Our most agentic Sonnet yet, with top tier intelligence for coding and everyday professional work.", "retrieved_at": "2026-09-12T07:47:51Z", "verified": true, "verified_at": "2026-09-12T07:47:51Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 1462, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "48 35 65 claude-sonnet-5-high Anthropic · Proprietary 1462 ±5 31,387", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "openai-gpt-5.6", "lab": "openai", "name": "GPT-5.6 Sol", "family": "GPT", "date": "2026-07-09", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/gpt-5-6/", "title": "GPT-5.6: Frontier intelligence that scales with your ambition", "quote": "GPT‑5.6 is available starting today across ChatGPT, Codex, and the OpenAI API.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "gpqa-diamond", "value": 94.6, "config": "GPT-5.6 Sol, no tools", "note": "First value (94.6%) is GPT-5.6 Sol; table also includes Terra, Luna, GPT-5.5 and third-party models for comparison.", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-5-6/", "quote": "GPQA Diamond 94.6%92.9%92.3%93.6%94.1%94.6%92.6%92%94.3%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "mmmu", "value": 83, "config": "MMMU-Pro, no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-5-6/", "quote": "MMMU Pro (no tools)83%80.7%78.4%81.2%——80.5%", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "lmarena-text", "value": 1483, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "17 9 32 gpt-5.6-sol-xhigh OpenAI · Proprietary 1483 ±5 23,413", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "GPT-5.6 ships as a family (Sol/Terra/Luna); Sol is the flagship recorded here. Launch post's Terminal-Bench score is versioned 2.1, not the basket's 2.0, so it is intentionally omitted for cross-release comparability. General-availability date (July 9, 2026) is not stated in the post's own text (which only carries later 'Update on ...' price-change notes); corroborated by Engadget (https://www.engadget.com/2210308/openai-rolls-out-gpt5-6-july-9/) and CNBC (https://www.cnbc.com/2026/07/08/openai-expanding-gpt-5point6-ai-model-release-ending-government-limits.html)." }, { "id": "xai-grok-4.5", "lab": "xai", "name": "Grok 4.5", "family": "Grok", "date": "2026-07-16", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok-4-5", "title": "Introducing Grok 4.5", "quote": "Today, we're launching Grok 4.5, SpaceXAI's smartest model built to excel at coding, agentic tasks, and knowledge work.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" }, "scores": [ { "benchmark": "swe-bench-pro", "value": 64.7, "config": "resolve rate", "reported_by": "official", "source": { "url": "https://x.ai/news/grok-4-5", "title": "Introducing Grok 4.5", "quote": "SWE Bench Pro resolve rate: Fable (max) 80.4%, Opus 4.8 (max) 69.2%, Grok 4.5 64.7%, Opus 4.7 (max) 64.3%, GLM 5.2 62.1%, GPT 5.5 (xhigh) 58.6%.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" } }, { "benchmark": "lmarena-text", "value": 1471, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "38 20 51 grok-4.5 SpaceXAI · Proprietary 1471 ±5 26,158", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "xAI's parent company is now named SpaceXAI following SpaceX's Feb 2026 acquisition of xAI. Launch benchmarks focus on DeepSWE, SWE Marathon, Terminal-Bench 2.1 and SWE-bench Pro; only SWE-bench Pro maps to a tracked (non-index) benchmark in this dataset. Terminal-Bench 2.1 is a point release of the Terminal-Bench 2.0 generation but is not recorded here since the basket's preferred_config specifies version 2.0 only." }, { "id": "google-gemini-4", "lab": "google", "name": "Gemini 4", "family": "Gemini Pro", "date": "2026-07-21", "date_precision": "day", "status": "announced", "announcement": { "url": "https://techcrunch.com/2026/07/21/google-releases-three-new-gemini-models-but-no-3-5-pro/", "title": "Google releases three new Gemini models — but no 3.5 Pro", "quote": "He also noted that the team has started its most ambitious pre-training run yet for Gemini 4.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:35:15Z" }, "scores": [], "notes": "Confirmed by Google DeepMind product lead Logan Kilpatrick (via X, reported by TechCrunch) on Jul 21, 2026: pre-training has begun, described by Sundar Pichai on the Q2 2026 earnings call (Jul 23) as needing to be 'significantly larger' than Gemini 3 to compete with rivals, with coding and autonomous agents named as priorities. 'Gemini 4' is the working/rumored name; no architecture, benchmark, price, access plan or release window has been disclosed, so date_precision is 'day' only for the announcement itself and no expected_window is recorded." }, { "id": "anthropic-claude-opus-5", "lab": "anthropic", "name": "Claude Opus 5", "family": "Claude Opus", "date": "2026-07-24", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://www.anthropic.com", "title": "Claude Fable 5.1", "quote": "Introducing Opus 5 Opus 5 is a step change for the Opus tier: stronger coding, more capable agents, and sharper professional work.", "retrieved_at": "2026-09-12T07:47:51Z", "verified": true, "verified_at": "2026-09-12T07:47:51Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 1493, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "9 4 17 claude-opus-5-high Anthropic · Proprietary 1493 ±5 35,174", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "moonshot-kimi-k3", "lab": "moonshot", "name": "Kimi K3", "family": "Kimi K", "date": "2026-07-27", "date_precision": "day", "status": "released", "announcement": { "url": "https://huggingface.co/moonshotai/Kimi-K3", "title": "moonshotai/Kimi-K3 model card", "quote": "Kimi K3 is an open-weight, native multimodal agentic model and our most capable model to date.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Moonshot announced K3 on Jul 16, 2026; open weights followed on Jul 27, 2026 (the date recorded here, per the project's rule of using public/usable availability)." }, "scores": [ { "benchmark": "gpqa-diamond", "value": 93.5, "config": "max reasoning effort", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K3", "title": "moonshotai/Kimi-K3 model card", "quote": "GPQA Diamond 93.5 92.6 94.1 91.0 93.5 91.2", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Table columns: Kimi K3 (max), Claude Fable 5 (max, w/ fallback), GPT-5.6 Sol (max), Claude Opus 4.8 (max), GPT-5.5 (xhigh), GLM-5.2 (max)." } }, { "benchmark": "hle", "value": 43.5, "config": "HLE-Full, no tools, max reasoning effort", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K3", "title": "moonshotai/Kimi-K3 model card", "quote": "HLE-Full 43.5 / 56.0 53.3 / 63.0 44.5 / 58.0 49.8 / 57.9 41.4 / 52.2", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:31:20Z", "note": "Cell format is 'without tools / with tools'; 43.5 is the no-tools figure for Kimi K3." } }, { "benchmark": "mmmu", "value": 81.6, "config": "MMMU-Pro, no tools, max reasoning effort", "reported_by": "official", "source": { "url": "https://huggingface.co/moonshotai/Kimi-K3", "title": "moonshotai/Kimi-K3 model card", "quote": "MMMU-Pro\n", "retrieved_at": "2026-09-07T04:20:00Z", "verified": false, "verified_at": "2026-09-07T09:31:20Z", "note": "Cell format is 'without tools / with tools' per the README's evaluation notes; 81.6 is the no-tools figure for Kimi K3 (first column)." } }, { "benchmark": "lmarena-text", "value": 1489, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "12 5 24 kimi-k3-max Moonshot · Kimi K3 license 1489 ±5 18,090", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Terminal-Bench 2.1 is also reported (Kimi K3 88.3%) but excluded per this project's basket rule requiring Terminal-Bench version 2.0 specifically. GPQA Diamond, HLE-Full and MMMU-Pro are the only tracked-basket benchmarks in the README's comparison table; no AIME, MMLU-Pro, LiveCodeBench, SWE-bench Verified, ARC-AGI-2 or τ2-bench figures are reported for this release." }, { "id": "google-gemini-3.7-flash", "lab": "google", "name": "Gemini 3.7 Flash", "family": "Gemini", "date": "2026-08-01", "date_precision": "month", "status": "released", "tier": "mid", "announcement": { "url": "https://deepmind.google", "title": "Gemini 4 Argon", "quote": "Introducing Gemini 3.7 Flash August 2026 Models", "retrieved_at": "2026-10-04T19:19:32Z", "verified": true, "verified_at": "2026-10-04T19:19:32Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 1491, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "11 3 25 gemini-3.7-flash-high Google · Proprietary 1491 ±8 Preliminary 5,682", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "alibaba-qwen3.8-max", "lab": "alibaba", "name": "Qwen3.8-Max", "family": "Qwen", "date": "2026-08-03", "date_precision": "day", "status": "released", "announcement": { "url": "https://qwen.ai/blog?id=qwen3.8", "title": "Qwen3.8-Max: A New Bar for Coding and Cowork", "quote": "Today, we are officially releasing Qwen 3.8-Max, the most capable model in the Qwen family to date.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "via": "r.jina.ai" }, "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "title": "Qwen/Qwen3.8-2.4T-A95B model card", "quote": "For the first time, Qwen3.8 brings a Qwen-Max-class model to open release.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z" } ], "scores": [ { "benchmark": "gpqa-diamond", "value": 92.6, "reported_by": "official", "source": { "url": "https://qwen.ai/blog?id=qwen3.8", "title": "Qwen3.8-Max: A New Bar for Coding and Cowork", "quote": "| GPQA Diamond | 92.0 | 92.6 | 94.1 | 92.4 | 92.6 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "via": "r.jina.ai", "note": "Table columns: Opus 4.8, Fable 5, GPT 5.6 Sol (max), Qwen3.7-Max, Qwen3.8-Max." } }, { "benchmark": "hle", "value": 43.6, "config": "no tools", "reported_by": "official", "source": { "url": "https://qwen.ai/blog?id=qwen3.8", "title": "Qwen3.8-Max: A New Bar for Coding and Cowork", "quote": "| HLE | 45.7 | 53.3 | 47.2 | 41.4 | 43.6 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "via": "r.jina.ai" } }, { "benchmark": "mmmu", "value": 82.3, "config": "MMMU-Pro", "reported_by": "official", "source": { "url": "https://qwen.ai/blog?id=qwen3.8", "title": "Qwen3.8-Max: A New Bar for Coding and Cowork", "quote": "| MMMU-Pro | 75.6 | 81.2 | 80.5 | 83.0 | 79.0 | 82.3 |", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "via": "r.jina.ai", "note": "Table columns: Opus4.8, Fable5, Gemini3.1-Pro, GPT5.6-Sol, Qwen3.7-Plus, Qwen3.8-Max." } }, { "benchmark": "lmarena-text", "value": 1480, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "22 10 39 qwen3.8-max Alibaba · Proprietary 1480 ±6 13,346", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "meta-muse-spark-1.2", "lab": "meta", "name": "Muse Spark 1.2", "family": "Muse", "date": "2026-08-05", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://ai.meta.com", "title": "Muse Wonder Full", "quote": "Muse Spark 1.2 brings a terminal coding agent, Muse Code that can plan, implement, and validate complex, multi-file changes across large repositories.", "retrieved_at": "2026-10-17T00:27:11Z", "verified": true, "verified_at": "2026-10-17T00:27:11Z", "via": "r.jina.ai" }, "scores": [] }, { "id": "xai-grok-4.6", "lab": "xai", "name": "Grok 4.6", "family": "Grok", "date": "2026-08-12", "date_precision": "day", "status": "released", "announcement": { "url": "https://x.ai/news/grok-4-6", "title": "Introducing Grok 4.6", "quote": "Today we are releasing Grok 4.6. Grok 4.6 builds on Grok 4.5 with a particular focus on long-running agents and more ambitious interactive and visual work.", "retrieved_at": "2026-09-07T07:45:00Z", "verified": true, "verified_at": "2026-09-07T09:30:58Z" }, "scores": [ { "benchmark": "lmarena-text", "value": 1461, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "49 27 72 grok-4.6-high SpaceXAI · Proprietary 1461 ±10 Preliminary 3,453", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Launch benchmarks (AA Intelligence Index, GDPVal-AA, CursorBench v3.2, DeepSWE v1.1, FrontierCode v1.1, APEX-Agents/APEX-SWE, Terminal-Bench v3.0, AA-Briefcase, Harvey LAB) are all third-party/aggregator evals not in this dataset's tracked benchmark basket (Terminal-Bench v3.0 is a newer, different-generation benchmark from the tracked 'Terminal-Bench 2.0'), so no scores are recorded for this release." }, { "id": "zhipu-glm-5.3", "lab": "zhipu", "name": "GLM-5.3", "family": "GLM", "date": "2026-08-14", "date_precision": "day", "status": "released", "announcement": { "url": "https://z.ai/blog/glm-5.3", "title": "GLM-5.3", "quote": "GLM-5.3 uses the same base model as GLM-5.2 — every gain comes from post-training.", "retrieved_at": "2026-09-07T03:26:49Z", "verified": false, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai" }, "scores": [ { "benchmark": "hle", "value": 62.5, "config": "with tools", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.3", "title": "GLM-5.3", "quote": "| HLE w/ Tools | 62.5 | 54.7 | 59.8 | 60.0 | 56.2 | 57.9 | 63.9 | 64.5 |", "retrieved_at": "2026-09-07T03:26:49Z", "verified": true, "verified_at": "2026-09-07T09:36:07Z", "via": "r.jina.ai", "note": "Row order: GLM-5.3, GLM-5.2, Kimi K3, DeepSeek-V4 Pro-0813, Qwen3.8-Max, Opus 4.8, Fable 5 (w/ fallback), GPT-5.6 Sol." } }, { "benchmark": "lmarena-text", "value": 1482, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "20 7 38 glm-5.3-max Z.ai · MIT 1482 ±7 7,668", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } }, { "benchmark": "terminal-bench-2", "value": 28.3, "config": "Claude Code 2.1.207 harness, reasoning effort=max, 400K context, 128K maximum output, avg@3", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.3", "title": "GLM-5.3", "quote": "GLM-5.3 improves from 4.6 to 28.3 on Terminal-Bench 3.0", "retrieved_at": "2026-09-09T05:58:51Z", "verified": true, "verified_at": "2026-09-09T05:58:51Z", "via": "r.jina.ai" } }, { "benchmark": "terminal-bench-2", "value": 88.2, "config": "Claude Code 2.1.207, temperature=1.0, top_p=1, max_new_tokens=65536, 6h timeout", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.3", "title": "GLM-5.3", "quote": "Terminal Bench 2.1 | 88.2", "retrieved_at": "2026-09-14T08:10:54Z", "verified": true, "verified_at": "2026-09-14T08:10:54Z", "via": "r.jina.ai" } }, { "benchmark": "mmmu", "value": 62.5, "config": "HLE w/ Tools", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.3", "title": "GLM-5.3", "quote": "HLE w/ Tools | 62.5", "retrieved_at": "2026-09-14T08:10:54Z", "verified": true, "verified_at": "2026-09-14T08:10:54Z", "via": "r.jina.ai" } }, { "benchmark": "hle", "value": 62.5, "config": "text-only", "reported_by": "official", "source": { "url": "https://z.ai/blog/glm-5.3", "title": "GLM-5.3", "quote": "HLE w/ Tools | 62.5", "retrieved_at": "2026-09-16T09:51:32Z", "verified": true, "verified_at": "2026-09-16T09:51:32Z", "via": "r.jina.ai" } } ], "notes": "GLM-5.3 is a post-training-only update on the GLM-5.2 base model; the launch post re-reports only coding/agentic/cyber benchmarks and HLE-with-tools, not the general reasoning basket (GPQA, plain HLE, AIME)." }, { "id": "alibaba-qwen4", "lab": "alibaba", "name": "Qwen4", "family": "Qwen", "date": "2026-08-24", "date_precision": "day", "status": "announced", "announcement": { "url": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "title": "Qwen/Qwen3.8-Flash-Next model card", "quote": "This experimental preview of the architecture that will underpin Qwen4 is built around a fundamental rethinking of how the core components of modern large language models (LLMs) interact at scale.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:40Z", "note": "Date is the Hugging Face repo creation date for Qwen3.8-Flash-Next, the model card that names Qwen4." }, "scores": [], "notes": "Alibaba has not published a dedicated Qwen4 announcement, name-confirmed spec, benchmark score, or expected release date. This entry tracks the lab's own acknowledgment (via the Qwen3.8-Flash-Next model card, described as an architecture preview for Qwen4) that a Qwen4 generation is in development. Press speculation (e.g. a rumored September 2026 window sourced to a single leak) is not lab-confirmed and is deliberately omitted from expected_window." }, { "id": "alibaba-qwen3.8-flash-next", "lab": "alibaba", "name": "Qwen3.8-Flash-Next", "family": "Qwen", "date": "2026-08-26", "date_precision": "day", "status": "released", "tier": "small", "announcement": { "url": "https://qwen.ai", "title": "Qwen3.8-Flash-Next", "quote": "In this release we are opening the weights of Qwen3.8-Flash-Next, a multimodal MoE model that also serves as an early preview of the architecture used in Qwen4.", "retrieved_at": "2026-09-10T06:58:20Z", "verified": true, "verified_at": "2026-09-10T06:58:20Z", "via": "r.jina.ai" }, "scores": [] }, { "id": "anthropic-claude-fable-5.1", "lab": "anthropic", "name": "Claude Fable 5.1", "family": "Claude Fable", "date": "2026-09-01", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "title": "Introducing Claude Fable 5.1 and Claude Mythos 5.1", "quote": "Claude Fable 5.1 is available today on all platforms, including Amazon Web Services, Google Cloud, and Microsoft Azure.", "retrieved_at": "2026-09-07T05:00:00Z", "verified": true, "verified_at": "2026-09-07T09:34:20Z" }, "sources": [ { "url": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20%26%20Claude%20Mythos%205.1%20System%20Card.pdf", "title": "System Card: Claude Fable 5.1 & Claude Mythos 5.1", "retrieved_at": "2026-09-07T05:00:00Z" } ], "scores": [ { "benchmark": "arc-agi-2", "value": 90, "config": "ARC Prize verified set", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20%26%20Claude%20Mythos%205.1%20System%20Card.pdf", "title": "System Card: Claude Fable 5.1 & Claude Mythos 5.1", "quote": "ARC-AGI-2 90.0% 89.2% 90.42% 92.5%", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 8.1.A, columns Fable 5.1/Mythos 5.1 (shared) / Fable 5/Mythos 5 (previous gen) / Claude Opus 5 / GPT-5.6 Sol; 90.0% is Fable 5.1's (and Mythos 5.1's) value." } }, { "benchmark": "hle", "value": 60.9, "config": "no tools", "reported_by": "official", "source": { "url": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20%26%20Claude%20Mythos%205.1%20System%20Card.pdf", "title": "System Card: Claude Fable 5.1 & Claude Mythos 5.1", "quote": "No tools 60.9% 57.8% 56.6% -", "retrieved_at": "2026-09-07T05:00:00Z", "verified": false, "verified_at": "2026-09-07T09:34:20Z", "note": "Table 8.1.A Humanity's Last Exam row, no-tools variant." } }, { "benchmark": "lmarena-text", "value": 1504, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "3 1 14 claude-fable-5.1-max Anthropic · Proprietary 1504 ±11 2,906", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ], "notes": "Unlike the 5.0 generation, Anthropic's Table 8.1.A for this release reports a single shared column for 'Claude Fable 5.1/Mythos 5.1' (the two are explicitly 'the same model, but with different levels of safeguards'), so scores above apply to both; Mythos 5.1 is therefore not recorded as a separate entry. Mythos 5.1 remains restricted to vetted cyberdefense/life-sciences partners via trusted access programs (not public). The same comparison table lists a 'Claude Opus 5' column (system card dated July 24, 2026 per public reporting), confirming the Opus numbered line continued past 4.8 in parallel with the Fable/Mythos lineage; it is not separately researched/recorded in this file given the time budget for this pass. GPQA Diamond, MMLU, MATH and HumanEval are no longer reported by Anthropic for this generation (GPQA was declared saturated as of the Fable 5/Mythos 5 system card); Terminal-Bench has moved to versions 4.0 and 'Science 0.1', neither comparable to this basket's Terminal-Bench 2.0, so no terminal-bench-2 score is recorded." }, { "id": "anthropic-claude-mythos-5.1", "lab": "anthropic", "name": "Claude Mythos 5.1", "family": "Claude Mythos", "date": "2026-09-01", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://www.anthropic.com", "title": "Claude Fable 5.1", "quote": "Introducing Fable 5.1 and Mythos 5.1 Our most advanced models for coding and knowledge work.", "retrieved_at": "2026-09-12T07:47:51Z", "verified": true, "verified_at": "2026-09-12T07:47:51Z" }, "scores": [] }, { "id": "google-gemini-3.8-flash", "lab": "google", "name": "Gemini 3.8 Flash", "family": "Gemini", "date": "2026-09-02", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://deepmind.google", "title": "Gemini 3.8 Flash", "quote": "Introducing Gemini 3.8 Flash and 3.8 Flash Cyber September 2026 Models", "retrieved_at": "2026-09-08T04:56:39Z", "verified": true, "verified_at": "2026-09-08T04:56:39Z" }, "sources": [ { "url": "https://deepmind.google/blog/introducing-gemini-3-8-flash-and-38-flash-cyber/", "title": "Gemini 3.8 Flash", "quote": "Building on the momentum of 3.7 Flash from three weeks ago and marking our third Flash release in only six weeks, today we’re introducing Gemini 3.8, our best reasoning and coding model yet, at the same speed and low cost of 3.7.", "retrieved_at": "2026-09-10T06:58:20Z", "verified": true, "verified_at": "2026-09-10T06:58:20Z" } ], "scores": [ { "benchmark": "hle", "value": 54.9, "config": "HLE-Verified", "reported_by": "official", "source": { "url": "https://deepmind.google/blog/introducing-gemini-3-8-flash-and-38-flash-cyber/", "title": "Gemini 3.8 Flash", "quote": "3.8 Flash also achieves a 54.9% on HLE-Verified, demonstrating its ability to handle multi-step reasoning across STEM, humanities, and professional fields.", "retrieved_at": "2026-09-10T06:58:20Z", "verified": true, "verified_at": "2026-09-10T06:58:20Z" } }, { "benchmark": "lmarena-text", "value": 1494, "config": "text, style control", "reported_by": "maintainer", "source": { "url": "https://lmarena.ai/leaderboard/text", "title": "LMArena text leaderboard", "quote": "8 1 22 gemini-3.8-flash-high Google · Proprietary 1494 ±9 Preliminary 5,125", "retrieved_at": "2026-10-12T05:44:11Z", "verified": true, "verified_at": "2026-10-12T05:44:11Z" } } ] }, { "id": "google-gemini-3.8-flash-cyber", "lab": "google", "name": "Gemini 3.8 Flash Cyber", "family": "Gemini", "date": "2026-09-02", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://deepmind.google/blog/introducing-gemini-3-8-flash-and-38-flash-cyber/", "title": "Gemini 3.8 Flash", "quote": "Building on the momentum of 3.7 Flash from three weeks ago and marking our third Flash release in only six weeks, today we’re introducing Gemini 3.8, our best reasoning and coding model yet, at the same speed and low cost of 3.7.", "retrieved_at": "2026-09-10T06:58:20Z", "verified": true, "verified_at": "2026-09-10T06:58:20Z" }, "scores": [ { "benchmark": "humaneval", "value": 47.2, "config": "CWE-Bench, pass@1", "reported_by": "official", "source": { "url": "https://deepmind.google/blog/introducing-gemini-3-8-flash-and-38-flash-cyber/", "title": "Gemini 3.8 Flash", "quote": "On this benchmark, Gemini 3.8 Flash Cyber is on the Pareto frontier: with a pass@1 of 47.2% compared to a leading frontier model at 47.8%, yet offered at a significantly lower cost.", "retrieved_at": "2026-09-10T06:58:20Z", "verified": true, "verified_at": "2026-09-10T06:58:20Z" } } ], "notes": "Available to trusted defenders through the Fairwind Program." }, { "id": "meta-muse-spark-1.3", "lab": "meta", "name": "Muse Spark 1.3", "family": "Muse", "date": "2026-09-02", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://ai.meta.com", "title": "Muse Spark 1.3", "quote": "Introducing Muse Spark 1.3 SEP 2, 2026", "retrieved_at": "2026-09-15T08:55:15Z", "verified": true, "verified_at": "2026-09-15T08:55:15Z", "via": "r.jina.ai" }, "scores": [] }, { "id": "openai-gpt-6-astra", "lab": "openai", "name": "GPT-6 Astra", "family": "GPT", "date": "2026-09-04", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra: A new generation of intelligence", "quote": "GPT‑6 Astra is rolling out today to a limited set of organizations and over the coming days will become available to all ChatGPT Plus, Pro, Business, and Enterprise users, as well as through the OpenAI API, Microsoft Azure, and AWS Bedrock.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" }, "scores": [ { "benchmark": "gpqa-diamond", "value": 96, "config": "no tools", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "quote": "GPQA Diamond tests graduate-level scientific reasoning in biology, chemistry, and physics. GPT‑6 Astra reaches a new high in the comparison shown at 96.0%.", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "arc-agi-2", "value": 95, "config": "ARC Prize verified set", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "quote": "ARC-AGI-2 95.0%92.5%90.0%89.2%90.4%-", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "hle", "value": 57.2, "config": "with tools", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "quote": "Humanity's Last Exam (w/ tools)57.2%-65.0%63.8%63.6%-", "retrieved_at": "2026-09-07T03:30:00Z", "verified": true, "verified_at": "2026-09-07T09:33:07Z" } }, { "benchmark": "terminal-bench-2", "value": 57.9, "config": "Terminal-Bench 4.0", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "Terminal-Bench 4.0 tests agents on complex terminal-based tasks, including software engineering, system configuration, and data analysis. GPT‑6 Astra reaches a new high at 57.9%", "retrieved_at": "2026-09-08T04:56:39Z", "verified": true, "verified_at": "2026-09-08T04:56:39Z" } }, { "benchmark": "arc-agi-2", "value": 95, "config": "ARC-AGI-2", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "ARC-AGI-2 95.0%", "retrieved_at": "2026-09-08T04:56:39Z", "verified": true, "verified_at": "2026-09-08T04:56:39Z" } }, { "benchmark": "mmmu", "value": 99.9, "config": "ARC-AGI-3", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "Astra also saturates ARC-AGI-3 with a 99.9% score", "retrieved_at": "2026-09-08T04:56:39Z", "verified": true, "verified_at": "2026-09-08T04:56:39Z" } }, { "benchmark": "browsecomp", "value": 91.5, "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "BrowseComp 91.5%", "retrieved_at": "2026-09-08T04:56:39Z", "verified": true, "verified_at": "2026-09-08T04:56:39Z" } }, { "benchmark": "osworld", "value": 72.6, "config": "OSWorld 2.0", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "In latency simulations on OSWorld 2.0, Astra achieves higher computer-use performance in about 47% less time per task than GPT‑5.6 Sol, scoring 72.6% at roughly 40 minutes per task", "retrieved_at": "2026-09-08T04:56:39Z", "verified": true, "verified_at": "2026-09-08T04:56:39Z" } }, { "benchmark": "gpqa-diamond", "value": 96, "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "GPQA Diamond tests graduate-level scientific reasoning in biology, chemistry, and physics. GPT‑6 Astra reaches a new high in the comparison shown at 96.0%.", "retrieved_at": "2026-09-09T05:58:51Z", "verified": true, "verified_at": "2026-09-09T05:58:51Z" } }, { "benchmark": "arc-agi-2", "value": 95, "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "ARC-AGI-2 95.0% 92.5% 90.0% 89.2% 90.4% -", "retrieved_at": "2026-09-09T05:58:51Z", "verified": true, "verified_at": "2026-09-09T05:58:51Z" } }, { "benchmark": "mmmu", "value": 72.6, "config": "OSWorld 2.0", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "In latency simulations on OSWorld 2.0, Astra achieves higher computer-use performance in about 47% less time per task than GPT‑5.6 Sol, scoring 72.6% at roughly 40 minutes per task", "retrieved_at": "2026-09-09T05:58:51Z", "verified": true, "verified_at": "2026-09-09T05:58:51Z" } }, { "benchmark": "tau2-bench", "value": 59.3, "config": "Agents' Last Exam", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "Agents’ Last Exam tests agents on complex professional tasks in real software, from financial modeling to engineering and media production. GPT‑6 Astra reaches a new high in the comparison shown, scoring 59.3%", "retrieved_at": "2026-09-09T05:58:51Z", "verified": true, "verified_at": "2026-09-09T05:58:51Z" } }, { "benchmark": "swe-bench-verified", "value": 74.1, "config": "DeepSWE v1.1", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "DeepSWE v1.1 74.1%", "retrieved_at": "2026-09-13T08:07:48Z", "verified": true, "verified_at": "2026-09-13T08:07:48Z" } }, { "benchmark": "mmlu-pro", "value": 99.9, "config": "ARC-AGI-3", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "ARC-AGI-3 99.9% 1", "retrieved_at": "2026-09-13T08:07:48Z", "verified": true, "verified_at": "2026-09-13T08:07:48Z" } }, { "benchmark": "hle", "value": 57.2, "config": "w/ tools", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "Humanity's Last Exam (w/ tools) 57.2%", "retrieved_at": "2026-09-13T08:07:48Z", "verified": true, "verified_at": "2026-09-13T08:07:48Z" } }, { "benchmark": "mmmu", "value": 92.7, "config": "ScreenSpot-Pro (no tools)", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "ScreenSpot-Pro (no tools) 92.7%", "retrieved_at": "2026-09-13T08:07:48Z", "verified": true, "verified_at": "2026-09-13T08:07:48Z" } }, { "benchmark": "osworld", "value": 72.6, "config": "OSWorld 2.0 (v2026.08.08, offline set, partial score)", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "OSWorld 2.0 (v2026.08.08, offline set, partial score) 72.6%", "retrieved_at": "2026-09-14T08:10:54Z", "verified": true, "verified_at": "2026-09-14T08:10:54Z" } }, { "benchmark": "mmlu-pro", "value": 99.9, "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "ARC-AGI-3 99.9% 1", "retrieved_at": "2026-09-15T08:55:15Z", "verified": true, "verified_at": "2026-09-15T08:55:15Z" } }, { "benchmark": "tau2-bench", "value": 59.3, "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "Agents’ Last Exam 59.3%", "retrieved_at": "2026-09-15T08:55:15Z", "verified": true, "verified_at": "2026-09-15T08:55:15Z" } }, { "benchmark": "mmmu", "value": 57.2, "config": "Humanity's Last Exam (w/ tools)", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "Humanity's Last Exam (w/ tools) 57.2%", "retrieved_at": "2026-09-16T09:51:32Z", "verified": true, "verified_at": "2026-09-16T09:51:32Z" } }, { "benchmark": "gpqa-diamond", "value": 96, "config": "no tools, pass@1", "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "GPQA Diamond tests graduate-level scientific reasoning in biology, chemistry, and physics. GPT‑6 Astra reaches a new high in the comparison shown at 96.0%.", "retrieved_at": "2026-09-18T11:14:58Z", "verified": true, "verified_at": "2026-09-18T11:14:58Z" } }, { "benchmark": "mmmu", "value": 0, "reported_by": "official", "source": { "url": "https://openai.com/index/gpt-6-astra/", "title": "GPT-6 Astra", "quote": "Astra also saturates ARC-AGI-3 with a 99.9% score and ExploitBench with a 100% score.", "retrieved_at": "2026-09-18T11:14:58Z", "verified": true, "verified_at": "2026-09-18T11:14:58Z" } } ], "notes": "Announced September 3, 2026 as a limited preview to trusted organizations; recorded date (September 4, 2026) is when OpenAI made a restricted version publicly available to paid ChatGPT users, per Al Jazeera (https://www.aljazeera.com/economy/2026/9/4/openai-unveils-gpt-6-astra-amid-rising-scrutiny-and-safety) and CNBC (https://www.cnbc.com/2026/09/03/open-ai-astra-gpt-6-cyber.html); not stated as a date in the announcement page's own body text. This is the newest confirmed OpenAI flagship as of this file's updated_at." }, { "id": "meta-muse", "lab": "meta", "name": "Muse", "family": "Muse", "date": "2026-09-08", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://ai.meta.com", "title": "Muse Wonder", "quote": "Introducing Muse: The World’s First Personal AI Agent Built for Everyone SEP 8, 2026", "retrieved_at": "2026-10-17T00:27:11Z", "verified": true, "verified_at": "2026-10-17T00:27:11Z", "via": "r.jina.ai" }, "scores": [] }, { "id": "anthropic-claude-mythos", "lab": "anthropic", "name": "Claude Mythos", "family": "Claude", "date": "2026-09-15", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://www.anthropic.com/news", "title": "Introducing Claude Mythos and Fable 5.1", "quote": "Introducing Claude Mythos and Fable 5.1", "retrieved_at": "2026-10-23T15:15:39Z", "verified": true, "verified_at": "2026-10-23T15:15:39Z" }, "scores": [] }, { "id": "anthropic-claude-opus-5.5", "lab": "anthropic", "name": "Claude Opus 5.5", "family": "Claude", "date": "2026-09-22", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://www.anthropic.com/news/claude-opus-5-5", "title": "Claude Opus 5.5", "quote": "We’re introducing Claude Opus 5.5, the first model in our new Claude 5.5 family.", "retrieved_at": "2026-09-27T15:42:05Z", "verified": true, "verified_at": "2026-09-27T15:42:05Z" }, "scores": [ { "benchmark": "terminal-bench-2", "value": 66.4, "config": "Terminal-Bench 4.0", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-opus-5-5", "title": "Claude Opus 5.5", "quote": "Agentic coding Terminal-Bench 4.0¹ 66.4%", "retrieved_at": "2026-09-27T15:42:05Z", "verified": true, "verified_at": "2026-09-27T15:42:05Z" } }, { "benchmark": "hle", "value": 67.7, "config": "with tools", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-opus-5-5", "title": "Claude Opus 5.5", "quote": "Multidisciplinary reasoning Humanity's Last Exam 67.7% with tools", "retrieved_at": "2026-09-27T15:42:05Z", "verified": true, "verified_at": "2026-09-27T15:42:05Z" } }, { "benchmark": "osworld", "value": 81.8, "config": "OSWorld 2.0", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-opus-5-5", "title": "Claude Opus 5.5", "quote": "Computer use OSWorld 2.0 81.8% partial", "retrieved_at": "2026-09-27T15:42:05Z", "verified": true, "verified_at": "2026-09-27T15:42:05Z" } }, { "benchmark": "osworld", "value": 81.8, "config": "OSWorld 2.1", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-opus-5-5", "title": "Claude Opus 5.5", "quote": "Computer use OSWorld 2.1 81.8% partial", "retrieved_at": "2026-10-19T02:12:51Z", "verified": true, "verified_at": "2026-10-19T02:12:51Z" } } ], "notes": "Terminal-Bench 4.0 results are reported for Claude Opus 5.5 at xhigh effort. GDPval-AA v2.1 is an Elo benchmark." }, { "id": "xai-grok-4.7", "lab": "xai", "name": "Grok 4.7", "family": "Grok", "date": "2026-09-22", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://x.ai/news", "title": "Introducing the SpaceX Super Intelligence Partner Network", "quote": "Sep 22, 2026\nIntroducing Grok 4.7\nSpaceXSI's most powerful model for coding and knowledge work.", "retrieved_at": "2026-10-23T15:15:39Z", "verified": true, "verified_at": "2026-10-23T15:15:39Z" }, "scores": [] }, { "id": "anthropic-claude-sonnet-5.5", "lab": "anthropic", "name": "Claude Sonnet 5.5", "family": "Claude Sonnet", "date": "2026-09-28", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://www.anthropic.com/news/claude-sonnet-5-5", "title": "Claude Sonnet 5.5", "quote": "Introducing Claude Sonnet 5.5, the second model in the Claude 5.5 family.", "retrieved_at": "2026-10-20T02:53:19Z", "verified": true, "verified_at": "2026-10-20T02:53:19Z" }, "scores": [ { "benchmark": "terminal-bench-2", "value": 70.6, "config": "Terminal-Bench 4.0", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-sonnet-5-5", "title": "Claude Sonnet 5.5", "quote": "Sonnet 5.5 scores 70.6% on Terminal-Bench 4.0, an agentic coding evaluation, compared to Sonnet 5’s 10.3%.", "retrieved_at": "2026-10-20T02:53:19Z", "verified": true, "verified_at": "2026-10-20T02:53:19Z" } }, { "benchmark": "hle", "value": 64.5, "config": "with tools", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-sonnet-5-5", "title": "Claude Sonnet 5.5", "quote": "Multidisciplinary reasoning Humanity’s Last Exam 64.5% with tools", "retrieved_at": "2026-10-20T02:53:19Z", "verified": true, "verified_at": "2026-10-20T02:53:19Z" } }, { "benchmark": "osworld", "value": 80.1, "config": "OSWorld 2.1, partial", "reported_by": "official", "source": { "url": "https://www.anthropic.com/news/claude-sonnet-5-5", "title": "Claude Sonnet 5.5", "quote": "Computer use OSWorld 2.1 80.1% partial", "retrieved_at": "2026-10-20T02:53:19Z", "verified": true, "verified_at": "2026-10-20T02:53:19Z" } } ], "notes": "Terminal-Bench 4.0 is used as the benchmark for terminal-bench-2. OSWorld 2.1 is used for osworld." }, { "id": "openai-gpt-6-luna", "lab": "openai", "name": "GPT-6 Luna", "family": "GPT", "date": "2026-09-29", "date_precision": "day", "status": "released", "tier": "small", "announcement": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Sol", "quote": "GPT‑6 Sol and GPT‑6 Luna are available in ChatGPT Work and Codex starting today for all Plus, Pro, Business, Enterprise, and Edu users.", "retrieved_at": "2026-09-29T17:15:04Z", "verified": true, "verified_at": "2026-09-29T17:15:04Z" }, "scores": [ { "benchmark": "swe-bench-verified", "value": 66.6, "config": "DeepSWE v1.1, max effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Sol", "quote": "GPT‑6 Luna at max effort scores 66.6%, comparable to Claude Opus 5 and Fable 5 at medium effort.", "retrieved_at": "2026-09-29T17:15:04Z", "verified": true, "verified_at": "2026-09-29T17:15:04Z" } }, { "benchmark": "swe-bench-verified", "value": 66.6, "config": "DeepSWE 1.1, max effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Sol", "quote": "GPT‑6 Luna at max effort scores 66.6%, comparable to Claude Opus 5 and Fable 5 at medium effort.", "retrieved_at": "2026-09-30T17:23:01Z", "verified": true, "verified_at": "2026-09-30T17:23:01Z" } }, { "benchmark": "swe-bench-verified", "value": 66.6, "config": "max effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Luna", "quote": "GPT‑6 Luna at max effort scores 66.6%, comparable to Claude Opus 5 and Fable 5 at medium effort.", "retrieved_at": "2026-09-30T17:23:01Z", "verified": true, "verified_at": "2026-09-30T17:23:01Z" } } ], "notes": "The page refers to this as a small-tier model relative to GPT-6 Astra." }, { "id": "openai-gpt-6-sol", "lab": "openai", "name": "GPT-6 Sol", "family": "GPT", "date": "2026-09-29", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Sol", "quote": "GPT‑6 Sol and GPT‑6 Luna are available in ChatGPT Work and Codex starting today for all Plus, Pro, Business, Enterprise, and Edu users.", "retrieved_at": "2026-09-29T17:15:04Z", "verified": true, "verified_at": "2026-09-29T17:15:04Z" }, "scores": [ { "benchmark": "swe-bench-verified", "value": 68.8, "config": "DeepSWE v1.1, max effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Sol", "quote": "On DeepSWE v1.1, which tests performance on complex software-engineering tasks in real codebases, GPT‑6 Sol at max effort scores 68.8%, within 1.1 percentage points of Claude Fable 5’s highest score in the evaluation—69.9% at xhigh effort—at approximately 80% lower cost per task.", "retrieved_at": "2026-09-29T17:15:04Z", "verified": true, "verified_at": "2026-09-29T17:15:04Z" } }, { "benchmark": "swe-bench-verified", "value": 68.8, "config": "DeepSWE 1.1, max effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Sol", "quote": "On DeepSWE v1.1, which tests performance on complex software-engineering tasks in real codebases, GPT‑6 Sol at max effort scores 68.8%, within 1.1 percentage points of Claude Fable 5’s highest score in the evaluation—69.9% at xhigh effort—at approximately 80% lower cost per task.", "retrieved_at": "2026-09-30T17:23:01Z", "verified": true, "verified_at": "2026-09-30T17:23:01Z" } }, { "benchmark": "osworld", "value": 60.5, "config": "OSWorld 2.0 offline, xhigh effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Sol", "quote": "On OSWorld 2.0 offline , GPT‑6 Sol at xhigh effort achieves a similar score to Claude Opus 5 at medium effort—60.5% versus 60.3%—at approximately 80% lower cost per task.", "retrieved_at": "2026-09-30T17:23:01Z", "verified": true, "verified_at": "2026-09-30T17:23:01Z" } }, { "benchmark": "swe-bench-verified", "value": 68.8, "config": "max effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Luna", "quote": "On DeepSWE v1.1, which tests performance on complex software-engineering tasks in real codebases, GPT‑6 Sol at max effort scores 68.8%, within 1.1 percentage points of Claude Fable 5’s highest score in the evaluation—69.9% at xhigh effort—at approximately 80% lower cost per task.", "retrieved_at": "2026-09-30T17:23:01Z", "verified": true, "verified_at": "2026-09-30T17:23:01Z" } }, { "benchmark": "osworld", "value": 60.5, "config": "xhigh effort, OSWorld 2.0 offline", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "title": "GPT-6 Luna", "quote": "On OSWorld 2.0 offline , GPT‑6 Sol at xhigh effort achieves a similar score to Claude Opus 5 at medium effort—60.5% versus 60.3%—at approximately 80% lower cost per task.", "retrieved_at": "2026-09-30T17:23:01Z", "verified": true, "verified_at": "2026-09-30T17:23:01Z" } } ], "notes": "The page refers to this as a mid-tier model relative to GPT-6 Astra." }, { "id": "openai-gpt-6.1-sol", "lab": "openai", "name": "GPT-6.1 Sol", "family": "GPT", "date": "2026-09-30", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://openai.com/index/introducing-gpt-6-1-sol/", "title": "GPT-6.1 Sol", "quote": "GPT‑6.1 Sol is available starting today to all Plus, Pro, Business, Enterprise, and Edu users in ChatGPT Work and Codex.", "retrieved_at": "2026-09-30T17:23:01Z", "verified": true, "verified_at": "2026-09-30T17:23:01Z" }, "sources": [ { "url": "https://openai.com/index/terminal-x", "title": "Terminal X matches senior investor judgment 88.6% with GPT-6.1", "quote": "OpenAI October 20, 2026 Startup Terminal X matches senior investor judgment 88.6% with GPT‑6.1", "retrieved_at": "2026-10-23T15:15:39Z", "verified": true, "verified_at": "2026-10-23T15:15:39Z" } ], "scores": [ { "benchmark": "terminal-bench-2", "value": 88.6, "config": "Eval Agent", "reported_by": "official", "source": { "url": "https://openai.com/index/terminal-x", "title": "Terminal X matches senior investor judgment 88.6% with GPT-6.1", "quote": "Across 500 scored judgments, its Eval Agent on GPT‑6.1 Sol agreed with the senior human reviewer 88.6% of the time", "retrieved_at": "2026-10-23T15:15:39Z", "verified": true, "verified_at": "2026-10-23T15:15:39Z" } } ], "notes": "The page describes GPT-6.1 Sol as an upgrade to GPT-6 Sol and compares it to the flagship GPT-6 Astra, positioning it as a mid-tier model." }, { "id": "google-gemini-4-carbon", "lab": "google", "name": "Gemini 4 Carbon", "family": "Gemini", "date": "2026-10-01", "date_precision": "month", "status": "announced", "tier": "flagship", "announcement": { "url": "https://deepmind.google", "title": "Gemini 4 Carbon", "quote": "Gemini 4 Carbon: Our next frontier model\nOctober 2026 Models", "retrieved_at": "2026-10-29T06:57:05Z", "verified": true, "verified_at": "2026-10-29T06:57:05Z" }, "scores": [] }, { "id": "google-nano-banana-2.1", "lab": "google", "name": "Nano Banana 2.1", "family": "Nano Banana", "date": "2026-10-01", "date_precision": "month", "status": "released", "tier": "small", "announcement": { "url": "https://blog.google/innovation-and-ai/products/gemini-app/new-tools-small-business-owners/", "title": "New Gemini tools for small business owners", "quote": "When paired with the power of Nano Banana 2.1 (our latest image generation model), Gemini can now deliver end-to-end visual results for new product launches or ongoing marketing campaigns.", "retrieved_at": "2026-10-23T15:15:39Z", "verified": true, "verified_at": "2026-10-23T15:15:39Z" }, "scores": [], "notes": "The page identifies this as an image generation model." }, { "id": "mistral-mistral-large-4", "lab": "mistral", "name": "Mistral Large 4", "family": "Mistral Large", "date": "2026-10-06", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://mistral.ai/news/mistral-large-4/", "title": "Mistral Large 4", "quote": "Today, we’re launching a public preview of Mistral Large 4.", "retrieved_at": "2026-10-06T20:42:11Z", "verified": true, "verified_at": "2026-10-06T20:42:11Z" }, "scores": [ { "benchmark": "terminal-bench-2", "value": 28.3, "config": "Terminal-Bench 4", "reported_by": "official", "source": { "url": "https://mistral.ai/news/mistral-large-4/", "title": "Mistral Large 4", "quote": "ML4 excels across software engineering, repository understanding, and complex terminal workflows, scoring 61.7% on DeepSWE v1.1, 59.4% on SWE-Atlas-QnA, and 28.3% on Terminal-Bench 4.", "retrieved_at": "2026-10-12T22:37:22Z", "verified": true, "verified_at": "2026-10-12T22:37:22Z" } }, { "benchmark": "terminal-bench-2", "value": 28.3, "reported_by": "official", "source": { "url": "https://mistral.ai/news/mistral-large-4/", "title": "Mistral Large 4", "quote": "ML4 excels across software engineering, repository understanding, and complex terminal workflows, scoring 61.7% on DeepSWE v1.1, 59.4% on SWE-Atlas-QnA, and 28.3% on Terminal-Bench 4.", "retrieved_at": "2026-10-13T23:26:17Z", "verified": true, "verified_at": "2026-10-13T23:26:17Z" } } ], "notes": "The page mentions the model is a 1 trillion-parameter natively multimodal model with 49 billion active parameters and is the lab's largest and most capable model to date." }, { "id": "openai-gpt-6", "lab": "openai", "name": "GPT-6", "family": "GPT", "date": "2026-10-07", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://openai.com/index/gpt-6-for-everyone", "title": "GPT-6 and Intelligent UI for everyone", "quote": "GPT‑6 with Intelligent UI starts rolling out globally to ChatGPT Plus, Pro, Business and Enterprise tiers today in the Chat tab.", "retrieved_at": "2026-10-18T00:12:50Z", "verified": true, "verified_at": "2026-10-18T00:12:50Z" }, "scores": [], "notes": "The page mentions GPT-6 Sol and GPT-6 Luna as models powering the release, but GPT-6 is the primary release name." }, { "id": "anthropic-claude-haiku-5.5", "lab": "anthropic", "name": "Claude Haiku 5.5", "family": "Claude Haiku", "date": "2026-10-07", "date_precision": "day", "status": "released", "tier": "small", "announcement": { "url": "https://www.anthropic.com", "title": "Claude Opus 5.5", "quote": "Oct 7, 2026 Introducing Claude Haiku 5.5", "retrieved_at": "2026-10-22T03:27:32Z", "verified": true, "verified_at": "2026-10-22T03:27:32Z" }, "scores": [] }, { "id": "meta-muse-wonder", "lab": "meta", "name": "Muse Wonder", "family": "Muse Wonder", "date": "2026-10-12", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://ai.meta.com", "title": "Muse Wonder", "quote": "We’re excited to introduce the Muse Wonder series with Muse Wonder and Muse Wonder Full, our frontier models that deliver state-of-the-art performance on personal and professional agentic tasks.", "retrieved_at": "2026-10-27T06:00:13Z", "verified": true, "verified_at": "2026-10-27T06:00:13Z", "via": "r.jina.ai" }, "scores": [] }, { "id": "meta-muse-wonder-full", "lab": "meta", "name": "Muse Wonder Full", "family": "Muse Wonder", "date": "2026-10-12", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://ai.meta.com", "title": "Muse Wonder", "quote": "We’re excited to introduce the Muse Wonder series with Muse Wonder and Muse Wonder Full, our frontier models that deliver state-of-the-art performance on personal and professional agentic tasks.", "retrieved_at": "2026-10-27T06:00:13Z", "verified": true, "verified_at": "2026-10-27T06:00:13Z", "via": "r.jina.ai" }, "scores": [] }, { "id": "openai-gpt-live-1", "lab": "openai", "name": "GPT-Live-1", "family": "GPT", "date": "2026-10-14", "date_precision": "day", "status": "released", "announcement": { "url": "https://openai.com/index/amp-beyond-coding", "title": "Amp puts AI agents to work beyond code with OpenAI models", "quote": "OpenAI October 14, 2026\nStartup\nAmp puts AI agents to work beyond code with OpenAI models", "retrieved_at": "2026-10-18T00:12:50Z", "verified": true, "verified_at": "2026-10-18T00:12:50Z" }, "scores": [] }, { "id": "xai-grok-voice-3.0", "lab": "xai", "name": "Grok Voice 3.0", "family": "Grok", "date": "2026-10-14", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://x.ai", "title": "Grok Voice 3.0", "quote": "Oct 14, 2026\nIntroducing Grok Voice 3.0", "retrieved_at": "2026-10-20T02:53:19Z", "verified": true, "verified_at": "2026-10-20T02:53:19Z" }, "scores": [] }, { "id": "deepseek-deepseek-v4.1-flash-1017", "lab": "deepseek", "name": "DeepSeek-V4.1-Flash-1017", "family": "DeepSeek-V4", "date": "2026-10-17", "date_precision": "day", "status": "released", "tier": "small", "announcement": { "url": "https://deepseek.com", "title": "DeepSeek-V4.1-Flash-1017", "quote": "DeepSeek V4.1 Flash 现已更新至 V4.1-Flash-1017 版本，项目开发能力提升显著，更低成本、更快速度完成任务；新增 API 支持网页搜索与抓取，欢迎测试和反馈", "retrieved_at": "2026-10-24T04:05:47Z", "verified": true, "verified_at": "2026-10-24T04:05:47Z" }, "scores": [] }, { "id": "google-gemini-4-argon", "lab": "google", "name": "Gemini 4 Argon", "family": "Gemini", "date": "2026-10-20", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://blog.google", "title": "Gemini 4 Argon", "quote": "Gemini 4 Argon is now available in the Gemini app, delivering frontier reasoning and improved performance across complex tasks.", "retrieved_at": "2026-10-20T02:53:19Z", "verified": true, "verified_at": "2026-10-20T02:53:19Z" }, "sources": [ { "url": "https://deepmind.google/models/gemini/", "title": "Gemini 4 Argon", "quote": "Gemini 4 Argon is available today for Google AI Pro and Ultra subscribers", "retrieved_at": "2026-10-27T06:00:13Z", "verified": true, "verified_at": "2026-10-27T06:00:13Z" } ], "scores": [ { "benchmark": "terminal-bench-2", "value": 57.4, "config": "Terminal-bench 4.0", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/", "title": "Gemini 4 Argon", "quote": "Terminal-bench 4.0 57.4%", "retrieved_at": "2026-10-27T06:00:13Z", "verified": true, "verified_at": "2026-10-27T06:00:13Z" } } ] }, { "id": "openai-gpt-6-astra-cyber", "lab": "openai", "name": "GPT-6 Astra Cyber", "family": "GPT", "date": "2026-10-23", "date_precision": "unknown", "status": "announced", "tier": "flagship", "announcement": { "url": "https://openai.com/index/securing-our-frontier-model-training-environment", "title": "Securing our frontier model training environment", "quote": "We used some of our most capable and aligned models, including our upcoming GPT‑6 Astra Cyber model and an internal frontier model.", "retrieved_at": "2026-10-24T00:02:36Z", "verified": true, "verified_at": "2026-10-24T00:02:36Z" }, "scores": [], "notes": "The page describes this as an 'upcoming' model used for red-teaming experiments." }, { "id": "moonshot-kimi-k3.1-pro", "lab": "moonshot", "name": "Kimi K3.1 Pro", "family": "Kimi", "date": "2026-10-24", "date_precision": "day", "status": "released", "tier": "flagship", "announcement": { "url": "https://moonshot.ai", "title": "Kimi K3.1 Pro", "quote": "2026-10-24 Kimi K3.1 Pro", "retrieved_at": "2026-11-01T08:36:58Z", "verified": true, "verified_at": "2026-11-01T08:36:58Z" }, "scores": [] }, { "id": "openai-gpt-6.1-terra", "lab": "openai", "name": "GPT-6.1 Terra", "family": "GPT", "date": "2026-10-27", "date_precision": "day", "status": "released", "tier": "mid", "announcement": { "url": "https://openai.com/index/introducing-gpt-6-1-terra/", "title": "GPT-6.1 Terra", "quote": "Starting today, GPT-6.1 Terra is available to all Plus, Pro, Enterprise, and Edu users in ChatGPT Work and Codex.", "retrieved_at": "2026-10-27T06:00:13Z", "verified": true, "verified_at": "2026-10-27T06:00:13Z" }, "scores": [ { "benchmark": "terminal-bench-2", "value": 50.3, "config": "maximum reasoning effort", "reported_by": "official", "source": { "url": "https://openai.com/index/introducing-gpt-6-1-terra/", "title": "GPT-6.1 Terra", "quote": "At maximum reasoning effort, GPT‑6.1 Terra solves 50.3% of tasks, more than double GPT‑5.6 Terra’s published score of 21.5%.", "retrieved_at": "2026-10-27T06:00:13Z", "verified": true, "verified_at": "2026-10-27T06:00:13Z" } } ], "notes": "The page mentions Terminal-Bench 4.0, but the benchmark list provided in the prompt only includes Terminal-Bench 2.0. The score is recorded under the requested benchmark ID." } ], "recent_changes": [ { "at": "2026-11-01T08:36:58Z", "actor": "worker", "lab": "moonshot", "release_id": "moonshot-kimi-k3.1-pro", "kind": "release_added", "summary": "Kimi K3.1 Pro — released 2026-10-24", "source_url": "https://moonshot.ai" }, { "at": "2026-10-29T06:57:05Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-4-carbon", "kind": "release_added", "summary": "Gemini 4 Carbon — announced 2026-10-01", "source_url": "https://deepmind.google" }, { "at": "2026-10-27T06:00:13Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-6.1-terra", "kind": "release_added", "summary": "GPT-6.1 Terra — released 2026-10-27", "source_url": "https://openai.com/index/introducing-gpt-6-1-terra/" }, { "at": "2026-10-27T06:00:13Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-4-argon", "kind": "score_added", "summary": "terminal-bench-2 57.4 (Terminal-bench 4.0)", "source_url": "https://deepmind.google/models/gemini/" }, { "at": "2026-10-27T06:00:13Z", "actor": "worker", "lab": "meta", "release_id": "meta-muse-wonder", "kind": "release_added", "summary": "Muse Wonder — released 2026-10-12", "source_url": "https://ai.meta.com" }, { "at": "2026-10-27T06:00:13Z", "actor": "worker", "lab": "meta", "release_id": "meta-muse-wonder-full", "kind": "release_added", "summary": "Muse Wonder Full — released 2026-10-12", "source_url": "https://ai.meta.com" }, { "at": "2026-10-25T04:52:24Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-deepseek-v4.1-flash-1017", "kind": "release_updated", "summary": "date 2026-10-01 (month) -> 2026-10-17 (day)", "source_url": "https://deepseek.com" }, { "at": "2026-10-24T04:05:47Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-deepseek-v4.1-flash-1017", "kind": "release_added", "summary": "DeepSeek-V4.1-Flash-1017 — released 2026-10-01", "source_url": "https://deepseek.com" }, { "at": "2026-10-24T00:02:36Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-6-astra-cyber", "kind": "release_added", "summary": "GPT-6 Astra Cyber — announced 2026-10-23", "source_url": "https://openai.com/index/securing-our-frontier-model-training-environment" }, { "at": "2026-10-23T15:15:39Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-6.1-sol", "kind": "score_added", "summary": "terminal-bench-2 88.6 (Eval Agent)", "source_url": "https://openai.com/index/terminal-x" }, { "at": "2026-10-23T15:15:39Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-mythos", "kind": "release_added", "summary": "Claude Mythos — released 2026-09-15", "source_url": "https://www.anthropic.com/news" }, { "at": "2026-10-23T15:15:39Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-haiku-4.5", "kind": "release_added", "summary": "Claude Haiku 4.5 — released 2025-12-01", "source_url": "https://www.anthropic.com/features/project-deal" }, { "at": "2026-10-23T15:15:39Z", "actor": "worker", "lab": "google", "release_id": "google-nano-banana-2.1", "kind": "release_added", "summary": "Nano Banana 2.1 — released 2026-10-01", "source_url": "https://blog.google/innovation-and-ai/products/gemini-app/new-tools-small-business-owners/" }, { "at": "2026-10-23T15:15:39Z", "actor": "worker", "lab": "xai", "release_id": "xai-grok-4.7", "kind": "release_added", "summary": "Grok 4.7 — released 2026-09-22", "source_url": "https://x.ai/news" }, { "at": "2026-10-22T03:27:32Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-haiku-5.5", "kind": "release_added", "summary": "Claude Haiku 5.5 — released 2026-10-07", "source_url": "https://www.anthropic.com" }, { "at": "2026-10-20T02:53:19Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-sonnet-5.5", "kind": "release_added", "summary": "Claude Sonnet 5.5 — released 2026-09-28", "source_url": "https://www.anthropic.com/news/claude-sonnet-5-5" }, { "at": "2026-10-20T02:53:19Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-4-argon", "kind": "release_added", "summary": "Gemini 4 Argon — released 2026-10-20", "source_url": "https://blog.google" }, { "at": "2026-10-20T02:53:19Z", "actor": "worker", "lab": "xai", "release_id": "xai-grok-voice-3.0", "kind": "release_added", "summary": "Grok Voice 3.0 — released 2026-10-14", "source_url": "https://x.ai" }, { "at": "2026-10-19T02:12:51Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-opus-5.5", "kind": "score_added", "summary": "osworld 81.8 (OSWorld 2.1)", "source_url": "https://www.anthropic.com/news/claude-opus-5-5" }, { "at": "2026-10-18T00:12:50Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-6", "kind": "release_added", "summary": "GPT-6 — released 2026-10-07", "source_url": "https://openai.com/index/gpt-6-for-everyone" }, { "at": "2026-10-18T00:12:50Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-live-1", "kind": "release_added", "summary": "GPT-Live-1 — released 2026-10-14", "source_url": "https://openai.com/index/amp-beyond-coding" }, { "at": "2026-10-17T00:27:11Z", "actor": "worker", "lab": "meta", "release_id": "meta-muse", "kind": "release_added", "summary": "Muse — released 2026-09-08", "source_url": "https://ai.meta.com" }, { "at": "2026-10-17T00:27:11Z", "actor": "worker", "lab": "meta", "release_id": "meta-muse-spark-1.2", "kind": "release_added", "summary": "Muse Spark 1.2 — released 2026-08-05", "source_url": "https://ai.meta.com" }, { "at": "2026-10-13T23:26:17Z", "actor": "worker", "lab": "mistral", "release_id": "mistral-mistral-large-4", "kind": "score_added", "summary": "terminal-bench-2 28.3", "source_url": "https://mistral.ai/news/mistral-large-4/" }, { "at": "2026-10-12T22:37:22Z", "actor": "worker", "lab": "mistral", "release_id": "mistral-mistral-large-4", "kind": "score_added", "summary": "terminal-bench-2 28.3 (Terminal-Bench 4)", "source_url": "https://mistral.ai/news/mistral-large-4/" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen3.8-max", "kind": "score_updated", "summary": "lmarena-text 1480 -> 1480", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen3-235b-a22b", "kind": "score_updated", "summary": "lmarena-text 1375 -> 1375", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen2.5-max", "kind": "score_updated", "summary": "lmarena-text 1374 -> 1374", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen2.5-72b", "kind": "score_updated", "summary": "lmarena-text 1303 -> 1303", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen2-72b", "kind": "score_updated", "summary": "lmarena-text 1261 -> 1261", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen1.5-110b", "kind": "score_updated", "summary": "lmarena-text 1234 -> 1234", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen-14b", "kind": "score_updated", "summary": "lmarena-text 1139 -> 1139", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-fable-5", "kind": "score_updated", "summary": "lmarena-text 1507 -> 1507", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-opus-4.6", "kind": "score_updated", "summary": "lmarena-text 1498 -> 1498", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-fable-5.1", "kind": "score_updated", "summary": "lmarena-text 1504 -> 1504", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-opus-5", "kind": "score_updated", "summary": "lmarena-text 1493 -> 1493", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-opus-4.8", "kind": "score_updated", "summary": "lmarena-text 1473 -> 1473", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-opus-4.5", "kind": "score_updated", "summary": "lmarena-text 1473 -> 1473", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-sonnet-5", "kind": "score_updated", "summary": "lmarena-text 1462 -> 1462", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-opus-4.1", "kind": "score_updated", "summary": "lmarena-text 1450 -> 1450", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-opus-4", "kind": "score_updated", "summary": "lmarena-text 1426 -> 1426", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-3.7-sonnet", "kind": "score_updated", "summary": "lmarena-text 1388 -> 1388", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-3.5-sonnet", "kind": "score_updated", "summary": "lmarena-text 1374 -> 1374", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-3-opus", "kind": "score_updated", "summary": "lmarena-text 1322 -> 1322", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-v4-pro", "kind": "score_updated", "summary": "lmarena-text 1458 -> 1458", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-v3.2", "kind": "score_updated", "summary": "lmarena-text 1425 -> 1425", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-v3.2-exp", "kind": "score_updated", "summary": "lmarena-text 1422 -> 1422", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-r1-0528", "kind": "score_updated", "summary": "lmarena-text 1421 -> 1421", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-v3.1", "kind": "score_updated", "summary": "lmarena-text 1417 -> 1417", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-r1", "kind": "score_updated", "summary": "lmarena-text 1398 -> 1398", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-v3-0324", "kind": "score_updated", "summary": "lmarena-text 1396 -> 1396", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-v3", "kind": "score_updated", "summary": "lmarena-text 1358 -> 1358", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-v2.5", "kind": "score_updated", "summary": "lmarena-text 1307 -> 1307", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "deepseek", "release_id": "deepseek-llm-67b", "kind": "score_updated", "summary": "lmarena-text 1184 -> 1184", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-3.8-flash", "kind": "score_updated", "summary": "lmarena-text 1494 -> 1494", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-3.7-flash", "kind": "score_updated", "summary": "lmarena-text 1491 -> 1491", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-3.1-pro", "kind": "score_updated", "summary": "lmarena-text 1487 -> 1487", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-3-pro", "kind": "score_updated", "summary": "lmarena-text 1486 -> 1486", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "google", "release_id": "google-gemini-2.5-pro", "kind": "score_updated", "summary": "lmarena-text 1446 -> 1446", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "google", "release_id": "google-palm-2", "kind": "score_updated", "summary": "lmarena-text 1138 -> 1138", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "meta", "release_id": "meta-muse-spark", "kind": "score_updated", "summary": "lmarena-text 1488 -> 1488", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "meta", "release_id": "meta-llama-3-70b", "kind": "score_updated", "summary": "lmarena-text 1276 -> 1276", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "meta", "release_id": "meta-llama-2-70b", "kind": "score_updated", "summary": "lmarena-text 1170 -> 1170", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "mistral", "release_id": "mistral-mistral-large-3", "kind": "score_updated", "summary": "lmarena-text 1414 -> 1414", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "mistral", "release_id": "mistral-mistral-7b", "kind": "score_updated", "summary": "lmarena-text 1110 -> 1110", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "moonshot", "release_id": "moonshot-kimi-k3", "kind": "score_updated", "summary": "lmarena-text 1489 -> 1489", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "moonshot", "release_id": "moonshot-kimi-k2.6", "kind": "score_updated", "summary": "lmarena-text 1461 -> 1461", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "moonshot", "release_id": "moonshot-kimi-k2.5", "kind": "score_updated", "summary": "lmarena-text 1451 -> 1451", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-5.6", "kind": "score_updated", "summary": "lmarena-text 1483 -> 1483", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-5.5", "kind": "score_updated", "summary": "lmarena-text 1477 -> 1477", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-5.4", "kind": "score_updated", "summary": "lmarena-text 1466 -> 1466", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-5.2", "kind": "score_updated", "summary": "lmarena-text 1436 -> 1436", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-5.1", "kind": "score_updated", "summary": "lmarena-text 1439 -> 1439", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-4.5", "kind": "score_updated", "summary": "lmarena-text 1445 -> 1445", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-5", "kind": "score_updated", "summary": "lmarena-text 1434 -> 1434", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-o1-preview", "kind": "score_updated", "summary": "lmarena-text 1388 -> 1388", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-4o", "kind": "score_updated", "summary": "lmarena-text 1346 -> 1346", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "openai", "release_id": "openai-gpt-4-turbo", "kind": "score_updated", "summary": "lmarena-text 1324 -> 1324", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "xai", "release_id": "xai-grok-4.5", "kind": "score_updated", "summary": "lmarena-text 1471 -> 1471", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "xai", "release_id": "xai-grok-4.1", "kind": "score_updated", "summary": "lmarena-text 1459 -> 1459", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "xai", "release_id": "xai-grok-4.6", "kind": "score_updated", "summary": "lmarena-text 1461 -> 1461", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "xai", "release_id": "xai-grok-2", "kind": "score_updated", "summary": "lmarena-text 1335 -> 1335", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-5.3", "kind": "score_updated", "summary": "lmarena-text 1482 -> 1482", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-5.2", "kind": "score_updated", "summary": "lmarena-text 1472 -> 1472", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-5.1", "kind": "score_updated", "summary": "lmarena-text 1466 -> 1466", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-5", "kind": "score_updated", "summary": "lmarena-text 1458 -> 1458", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-4.7", "kind": "score_updated", "summary": "lmarena-text 1442 -> 1442", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-4.6", "kind": "score_updated", "summary": "lmarena-text 1425 -> 1425", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-4.5", "kind": "score_updated", "summary": "lmarena-text 1411 -> 1411", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-glm-4-plus", "kind": "score_updated", "summary": "lmarena-text 1319 -> 1319", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-12T05:44:11Z", "actor": "worker", "lab": "zhipu", "release_id": "zhipu-chatglm-6b", "kind": "score_updated", "summary": "lmarena-text 995 -> 995", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-06T20:42:11Z", "actor": "worker", "lab": "mistral", "release_id": "mistral-mistral-large-4", "kind": "release_added", "summary": "Mistral Large 4 — released 2026-10-06", "source_url": "https://mistral.ai/news/mistral-large-4/" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen3.8-max", "kind": "score_updated", "summary": "lmarena-text 1480 -> 1480", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen3-235b-a22b", "kind": "score_updated", "summary": "lmarena-text 1375 -> 1375", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen2.5-max", "kind": "score_updated", "summary": "lmarena-text 1374 -> 1374", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen2.5-72b", "kind": "score_updated", "summary": "lmarena-text 1303 -> 1303", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen2-72b", "kind": "score_updated", "summary": "lmarena-text 1261 -> 1261", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen1.5-110b", "kind": "score_updated", "summary": "lmarena-text 1234 -> 1234", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "alibaba", "release_id": "alibaba-qwen-14b", "kind": "score_updated", "summary": "lmarena-text 1139 -> 1139", "source_url": "https://lmarena.ai/leaderboard/text" }, { "at": "2026-10-05T05:13:07Z", "actor": "worker", "lab": "anthropic", "release_id": "anthropic-claude-fable-5", "kind": "score_updated", "summary": "lmarena-text 1507 -> 1507", "source_url": "https://lmarena.ai/leaderboard/text" } ], "backtest": { "from": "2019-06-11", "to": "2026-11-06", "stepDays": 30, "n": 444, "unforecastable": 444, "coverage68": 0.6959459459459459, "coverage90": 0.8648648648648649, "maeDays": 112.33108108108108, "medianAbsDays": 56, "biasDays": -1.4977477477477477, "thetaMae": 0.9466238306800961, "byLab": { "openai": { "n": 90, "coverage68": 0.6111111111111112, "coverage90": 0.8222222222222222, "maeDays": 181.95555555555555 }, "anthropic": { "n": 43, "coverage68": 0.7441860465116279, "coverage90": 0.9069767441860465, "maeDays": 52.55813953488372 }, "google": { "n": 42, "coverage68": 0.9047619047619048, "coverage90": 1, "maeDays": 69.16666666666667 }, "xai": { "n": 36, "coverage68": 0.1111111111111111, "coverage90": 0.25, "maeDays": 287.4166666666667 }, "meta": { "n": 44, "coverage68": 0.7954545454545454, "coverage90": 0.9772727272727273, "maeDays": 85.79545454545455 }, "deepseek": { "n": 30, "coverage68": 0.9333333333333333, "coverage90": 0.9666666666666667, "maeDays": 34.36666666666667 }, "alibaba": { "n": 36, "coverage68": 0.7222222222222222, "coverage90": 0.8611111111111112, "maeDays": 64 }, "moonshot": { "n": 37, "coverage68": 0.7837837837837838, "coverage90": 1, "maeDays": 97.51351351351352 }, "zhipu": { "n": 49, "coverage68": 0.7551020408163265, "coverage90": 0.9591836734693877, "maeDays": 88 }, "mistral": { "n": 37, "coverage68": 0.6756756756756757, "coverage90": 0.8918918918918919, "maeDays": 79.91891891891892 } }, "calibration": [ { "nominal": 0.1, "observed": 0.06981981981981981 }, { "nominal": 0.2, "observed": 0.14864864864864866 }, { "nominal": 0.3, "observed": 0.28603603603603606 }, { "nominal": 0.4, "observed": 0.42342342342342343 }, { "nominal": 0.5, "observed": 0.5427927927927928 }, { "nominal": 0.6, "observed": 0.6351351351351351 }, { "nominal": 0.7, "observed": 0.722972972972973 }, { "nominal": 0.8, "observed": 0.8063063063063063 }, { "nominal": 0.9, "observed": 0.8941441441441441 } ], "sigmaScale": 1.3, "halfLifeDays": 730, "drift": true, "driftPerYear": -0.3727584342209985, "rows": [] }, "worker": { "last_run_at": "2026-11-06T21:33:43Z", "last_success_at": "2026-11-06T21:33:43Z", "pages_polled": 23, "pages_changed": 0, "llm_model": "google/gemini-3.1-flash-lite", "next_run_at": "2026-11-06T21:33:43Z", "run_status": "running", "run_step": "poll", "interval_minutes": 60, "last_run_summary": "poll: 23 pages, 0 changed, 0 LLM calls, 0 new releases · 0.000 USD", "researcher": { "version": "2.0.0", "last_backfill_at": "2026-11-04T20:39:22Z", "last_arena_at": "2026-10-12T05:44:11Z", "last_eval_at": "2026-11-04T20:39:29Z", "eval": { "evaluated_at": "2026-11-04T20:39:29Z", "gold_releases": 103, "found_releases": 3, "matched_releases": 1, "precision_releases": 0.3333, "recall_releases": 0.0097, "gold_scores": 418, "matched_scores": 0, "score_recall": 0, "score_mae": 0, "quotes_total": 1, "quotes_verified": 1, "quote_verified_rate": 1, "by_lab": { "openai": { "gold": 17, "found": 0, "matched": 0, "scores_gold": 63, "scores_matched": 0 }, "anthropic": { "gold": 14, "found": 3, "matched": 1, "scores_gold": 50, "scores_matched": 0 }, "google": { "gold": 9, "found": 0, "matched": 0, "scores_gold": 38, "scores_matched": 0 }, "xai": { "gold": 9, "found": 0, "matched": 0, "scores_gold": 25, "scores_matched": 0 }, "meta": { "gold": 7, "found": 0, "matched": 0, "scores_gold": 30, "scores_matched": 0 }, "deepseek": { "gold": 11, "found": 0, "matched": 0, "scores_gold": 60, "scores_matched": 0 }, "alibaba": { "gold": 11, "found": 0, "matched": 0, "scores_gold": 40, "scores_matched": 0 }, "moonshot": { "gold": 7, "found": 0, "matched": 0, "scores_gold": 33, "scores_matched": 0 }, "zhipu": { "gold": 11, "found": 0, "matched": 0, "scores_gold": 49, "scores_matched": 0 }, "mistral": { "gold": 7, "found": 0, "matched": 0, "scores_gold": 30, "scores_matched": 0 } }, "unverified_extras": [ { "lab": "anthropic", "name": "Claude Opus 5.5", "date": "2026-09-22", "url": "https://www.anthropic.com/news/claude-opus-5-5" }, { "lab": "anthropic", "name": "Claude Haiku 5.5", "date": "2026-10-07", "url": "https://www.anthropic.com/news" } ] }, "budget": { "calls": 0, "tokens_in": 0, "tokens_out": 0, "usd_estimate": 0 }, "cadence": { "tier": "weekly", "interval_hours": 168, "visitors_per_day": 1.67, "days_measured": 6, "capped": false, "next_research_at": "2026-11-11T20:39:22Z" }, "usage_total": { "calls": 1813, "tokens_in": 2723536, "tokens_out": 201493, "usd_estimate": 29.62238 }, "last_backfill_summary": "arena: 400 rows, 66 matched, 66 scores written to 10 lab files, 313 unmatched · backfill: 10 labs, 0 candidates, 0 releases / 0 scores, 0 calls · 0.00 USD — 100 errors · eval: 1/103 gold matched, precision 0.3333, recall 0.0097, score recall 0, 2 unverified extras → promote NOT MET" } } }
Humanity's Last Exam Academic reasoning (full set, text + MM) No tools44.4%37.5%GPQA Diamond Scientific knowledgeNo tools94.3%91.9%Terminal-Bench 2.0 Agentic terminal codingTerminus-2 harness68.5%56.9%SWE-Bench Verified Agentic codingSingle attempt80.6%76.2%Retail90.8%85.3%91.7%91.9%82.0%—
Telecom99.3%", "retrieved_at": "2026-09-07T04:09:50Z", "verified": false, "verified_at": "2026-09-07T09:35:15Z" } }, { "benchmark": "mmmu", "value": 80.5, "config": "MMMU-Pro, Thinking (High), no tools", "reported_by": "official", "source": { "url": "https://deepmind.google/models/gemini/pro/", "title": "Gemini Pro — Google DeepMind", "quote": "MMMU-Pro Multimodal understanding and reasoning No tools80.5%81.0%SWE-Bench Pro (Public) Diverse agentic coding tasks Single attempt54.2%43.3%81.6 / 83.4